Bound table proposals to complete captioned three-rule source regions

docs/MAINTAINERS.md

@@ -303,6 +303,16 @@ structured spans are authoritative. Never describe an accepted result as proof
 of the one uniquely correct or semantically correct topology. Rejections and
 ordinary conversions remain fail-closed and emit no inferred table.
 
+Proposal regions may use one uniquely captioned three-horizontal-fill-band
+envelope to recover complete source membership, including multiline heading
+text, while excluding nearby prose. This is region selection only: it does not
+infer columns, cell boundaries or header roles. All intersecting text must be
+accounted for; competing paint, incomplete geometry, ambiguous captions and
+boundary-straddling items preserve the original unrefined proposal. Refined
+regions suppress header hints. The same source region is used by document maps
+and fresh proposal verification. Ordinary Markdown and conversion gaps do not
+change, and matching source/region tokens do not excuse obsolete assignments.
+
 Filesystem operations and rasterization happen locally, and PDF Tools does not
 upload files to a separate PDF service. Text, images, and metadata returned
 through MCP may still be processed under the selected host or model provider's

pdf-toolkit-mcp-share/server/markdown-conversion.js

@@ -2815,15 +2815,92 @@ function tableProposalHeaderHints(run) {
   };
 }
 
+function threeRuleProposalEnvelope(page, run) {
+  // This refines an already abandoned proposal, never reconstructs a table.
+  // Three full-width paint bands and a nearby caption can bound the source
+  // region without establishing columns, cell cuts, or header semantics.
+  const evidence = page.ruled_rects;
+  if (evidence?.status !== "available" || page.truncation?.truncated
+    || evidence.items.length > MAX_TABLE_PROPOSAL_RULED_RECTS
+    || evidence.observed_count !== evidence.items.length
+    || evidence.returned_count !== evidence.items.length) return null;
+  if ([page.ruling_segments, page.painted_rectangles].some(channel => (
+    channel?.status !== "available" || channel.truncated
+    || channel.observed_count !== channel.items.length
+    || channel.returned_count !== channel.items.length
+  ))) return null;
+  const text = page.raw_items.filter(item => item.text_kind === "non_whitespace");
+  if (text.some(item => !item.geometry_valid || !item.bbox)) return null;
+  const tolerance = 0.05;
+  const bars = evidence.items.filter(rect => ["fill", "eoFill"].includes(rect.verb)
+    && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5)
+    .sort((a, b) => a.y - b.y || a.x - b.x);
+  const bands = [];
+  for (const bar of bars) {
+    const prior = bands.at(-1);
+    if (prior && Math.abs(prior.y - bar.y) <= tolerance
+      && Math.abs(prior.height - bar.height) <= tolerance
+      && Math.abs(prior.x + prior.width - bar.x) <= tolerance) {
+      prior.width = bar.x + bar.width - prior.x;
+      prior.parts.push(bar);
+    } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] });
+  }
+  const contains = (outer, inner) => inner.x >= outer.x && inner.y >= outer.y
+    && inner.x + inner.width <= outer.x + outer.width
+    && inner.y + inner.height <= outer.y + outer.height;
+  const runIds = new Set(run.map(row => row.line.id));
+  const candidates = [];
+  for (const top of bands) {
+    if (top.width < 80) continue;
+    const aligned = bands.filter(band => Math.abs(band.x - top.x) <= tolerance
+      && Math.abs(band.width - top.width) <= tolerance);
+    if (aligned.length !== 3 || aligned[0] !== top) continue;
+    const [, middle, bottom] = aligned;
+    if (middle.y - top.y - top.height < 5 || bottom.y - middle.y - middle.height < 5
+      || bottom.y + bottom.height - top.y > 300) continue;
+    const bbox = { x: top.x, y: top.y, width: top.width,
+      height: bottom.y + bottom.height - top.y };
+    const interior = { x: bbox.x, y: top.y + top.height, width: bbox.width,
+      height: bottom.y - top.y - top.height };
+    // Reject partial/competing paint inside this envelope, not just extra
+    // full-width rules. Clip boxes are retained as evidence, not paint bands.
+    const constituents = new Set(aligned.flatMap(band => band.parts));
+    if (evidence.items.some(rect => !["clip", "eoClip"].includes(rect.verb)
+      && !constituents.has(rect) && rectsOverlap(rect, bbox))
+      || page.ruling_segments.items.some(segment => rulingSegmentOverlapsRegion(segment, bbox))
+      || page.painted_rectangles.items.some(rect => rectsOverlap(rect.bbox, bbox))) continue;
+    const captions = page.lines.filter(line => /^Table\s+\d+\b/iu.test(line.text.trim())
+      && line.x >= bbox.x && line.x + line.width <= bbox.x + bbox.width
+      && line.y + line.height <= top.y
+      && top.y - line.y - line.height <= Math.min(48, 3 * line.height));
+    if (captions.length !== 1) continue;
+    const intersecting = text.filter(item => rectsOverlap(item.bbox, bbox));
+    if (intersecting.length === 0 || intersecting.some(item => !contains(interior, item.bbox)
+      || rectsOverlap(item.bbox, middle))) continue;
+    const lines = page.lines.filter(line => rectsOverlap(line, interior));
+    if (lines.some(line => !contains(interior, line))) continue;
+    // Reopen every retained source item, including lines missed by the old
+    // adjacent-run heuristic. Never silently drop an unassigned source item.
+    const itemIds = lines.flatMap(line => line.item_ids)
+      .filter(id => intersecting.some(item => item.id === id));
+    if (new Set(itemIds).size !== itemIds.length || itemIds.length !== intersecting.length) continue;
+    if (lines.filter(line => runIds.has(line.id)).length < 2
+      || !lines.some(line => line.y + line.height <= middle.y)
+      || !lines.some(line => line.y >= middle.y + middle.height)) continue;
+    const itemById = new Map(page.raw_items.map(item => [item.id, item]));
+    candidates.push({ bbox, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) });
+  }
+  return candidates.length === 1 ? candidates[0] : null;
+}
+
 /**
- * Build one deterministic, bounded proposal-region descriptor for a table run
- * the renderer abandoned. Pure over the page IR and the run; no token, no I/O,
- * no time/RNG. The proposal_token that binds it to (source sha, IR version,
- * region_id) is added by the handler, which owns the source identity. Exported
- * for the bounded-truncation regression, which needs to force an over-cap
- * region without synthesizing a full validated IR.
+ * Build a bounded proposal from an abandoned run. A uniquely captioned
+ * three-rule envelope can refine source membership but supplies no topology
+ * or header authority. Pure over source-validated IR; no token, time or I/O.
  */
 export function buildTableProposalRegion(page, run, reason, ordinal) {
+  const envelope = threeRuleProposalEnvelope(page, run);
+  if (envelope) run = envelope.run;
   const itemById = new Map(page.raw_items.map(item => [item.id, item]));
   let minX = Infinity;
   let minY = Infinity;
@@ -2835,7 +2912,7 @@ export function buildTableProposalRegion(page, run, reason, ordinal) {
     maxX = Math.max(maxX, line.x + line.width);
     maxY = Math.max(maxY, line.y + line.height);
   }
-  const bbox = { x: minX, y: minY, width: maxX - minX, height: maxY - minY };
+  const bbox = envelope?.bbox ?? { x: minX, y: minY, width: maxX - minX, height: maxY - minY };
 
   const allItems = [];
   for (const { line } of run) {
@@ -2895,7 +2972,10 @@ export function buildTableProposalRegion(page, run, reason, ordinal) {
     ruled_rects: ruledRects,
     ruling_segments: rulingSegments,
     painted_rectangles: paintedRectangles,
-    header_hints: tableProposalHeaderHints(run),
+    header_hints: envelope ? {
+      status: "unavailable", first_row_height: null, body_median_height: null,
+      first_row_band: "unavailable",
+    } : tableProposalHeaderHints(run),
     truncation: {
       text_items: itemsTruncated ? "truncated" : "complete",
       ruled_rects: ruledTruncated ? "truncated" : "complete",

server/markdown-conversion.js

@@ -2815,15 +2815,92 @@ function tableProposalHeaderHints(run) {
   };
 }
 
+function threeRuleProposalEnvelope(page, run) {
+  // This refines an already abandoned proposal, never reconstructs a table.
+  // Three full-width paint bands and a nearby caption can bound the source
+  // region without establishing columns, cell cuts, or header semantics.
+  const evidence = page.ruled_rects;
+  if (evidence?.status !== "available" || page.truncation?.truncated
+    || evidence.items.length > MAX_TABLE_PROPOSAL_RULED_RECTS
+    || evidence.observed_count !== evidence.items.length
+    || evidence.returned_count !== evidence.items.length) return null;
+  if ([page.ruling_segments, page.painted_rectangles].some(channel => (
+    channel?.status !== "available" || channel.truncated
+    || channel.observed_count !== channel.items.length
+    || channel.returned_count !== channel.items.length
+  ))) return null;
+  const text = page.raw_items.filter(item => item.text_kind === "non_whitespace");
+  if (text.some(item => !item.geometry_valid || !item.bbox)) return null;
+  const tolerance = 0.05;
+  const bars = evidence.items.filter(rect => ["fill", "eoFill"].includes(rect.verb)
+    && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5)
+    .sort((a, b) => a.y - b.y || a.x - b.x);
+  const bands = [];
+  for (const bar of bars) {
+    const prior = bands.at(-1);
+    if (prior && Math.abs(prior.y - bar.y) <= tolerance
+      && Math.abs(prior.height - bar.height) <= tolerance
+      && Math.abs(prior.x + prior.width - bar.x) <= tolerance) {
+      prior.width = bar.x + bar.width - prior.x;
+      prior.parts.push(bar);
+    } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] });
+  }
+  const contains = (outer, inner) => inner.x >= outer.x && inner.y >= outer.y
+    && inner.x + inner.width <= outer.x + outer.width
+    && inner.y + inner.height <= outer.y + outer.height;
+  const runIds = new Set(run.map(row => row.line.id));
+  const candidates = [];
+  for (const top of bands) {
+    if (top.width < 80) continue;
+    const aligned = bands.filter(band => Math.abs(band.x - top.x) <= tolerance
+      && Math.abs(band.width - top.width) <= tolerance);
+    if (aligned.length !== 3 || aligned[0] !== top) continue;
+    const [, middle, bottom] = aligned;
+    if (middle.y - top.y - top.height < 5 || bottom.y - middle.y - middle.height < 5
+      || bottom.y + bottom.height - top.y > 300) continue;
+    const bbox = { x: top.x, y: top.y, width: top.width,
+      height: bottom.y + bottom.height - top.y };
+    const interior = { x: bbox.x, y: top.y + top.height, width: bbox.width,
+      height: bottom.y - top.y - top.height };
+    // Reject partial/competing paint inside this envelope, not just extra
+    // full-width rules. Clip boxes are retained as evidence, not paint bands.
+    const constituents = new Set(aligned.flatMap(band => band.parts));
+    if (evidence.items.some(rect => !["clip", "eoClip"].includes(rect.verb)
+      && !constituents.has(rect) && rectsOverlap(rect, bbox))
+      || page.ruling_segments.items.some(segment => rulingSegmentOverlapsRegion(segment, bbox))
+      || page.painted_rectangles.items.some(rect => rectsOverlap(rect.bbox, bbox))) continue;
+    const captions = page.lines.filter(line => /^Table\s+\d+\b/iu.test(line.text.trim())
+      && line.x >= bbox.x && line.x + line.width <= bbox.x + bbox.width
+      && line.y + line.height <= top.y
+      && top.y - line.y - line.height <= Math.min(48, 3 * line.height));
+    if (captions.length !== 1) continue;
+    const intersecting = text.filter(item => rectsOverlap(item.bbox, bbox));
+    if (intersecting.length === 0 || intersecting.some(item => !contains(interior, item.bbox)
+      || rectsOverlap(item.bbox, middle))) continue;
+    const lines = page.lines.filter(line => rectsOverlap(line, interior));
+    if (lines.some(line => !contains(interior, line))) continue;
+    // Reopen every retained source item, including lines missed by the old
+    // adjacent-run heuristic. Never silently drop an unassigned source item.
+    const itemIds = lines.flatMap(line => line.item_ids)
+      .filter(id => intersecting.some(item => item.id === id));
+    if (new Set(itemIds).size !== itemIds.length || itemIds.length !== intersecting.length) continue;
+    if (lines.filter(line => runIds.has(line.id)).length < 2
+      || !lines.some(line => line.y + line.height <= middle.y)
+      || !lines.some(line => line.y >= middle.y + middle.height)) continue;
+    const itemById = new Map(page.raw_items.map(item => [item.id, item]));
+    candidates.push({ bbox, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) });
+  }
+  return candidates.length === 1 ? candidates[0] : null;
+}
+
 /**
- * Build one deterministic, bounded proposal-region descriptor for a table run
- * the renderer abandoned. Pure over the page IR and the run; no token, no I/O,
- * no time/RNG. The proposal_token that binds it to (source sha, IR version,
- * region_id) is added by the handler, which owns the source identity. Exported
- * for the bounded-truncation regression, which needs to force an over-cap
- * region without synthesizing a full validated IR.
+ * Build a bounded proposal from an abandoned run. A uniquely captioned
+ * three-rule envelope can refine source membership but supplies no topology
+ * or header authority. Pure over source-validated IR; no token, time or I/O.
  */
 export function buildTableProposalRegion(page, run, reason, ordinal) {
+  const envelope = threeRuleProposalEnvelope(page, run);
+  if (envelope) run = envelope.run;
   const itemById = new Map(page.raw_items.map(item => [item.id, item]));
   let minX = Infinity;
   let minY = Infinity;
@@ -2835,7 +2912,7 @@ export function buildTableProposalRegion(page, run, reason, ordinal) {
     maxX = Math.max(maxX, line.x + line.width);
     maxY = Math.max(maxY, line.y + line.height);
   }
-  const bbox = { x: minX, y: minY, width: maxX - minX, height: maxY - minY };
+  const bbox = envelope?.bbox ?? { x: minX, y: minY, width: maxX - minX, height: maxY - minY };
 
   const allItems = [];
   for (const { line } of run) {
@@ -2895,7 +2972,10 @@ export function buildTableProposalRegion(page, run, reason, ordinal) {
     ruled_rects: ruledRects,
     ruling_segments: rulingSegments,
     painted_rectangles: paintedRectangles,
-    header_hints: tableProposalHeaderHints(run),
+    header_hints: envelope ? {
+      status: "unavailable", first_row_height: null, body_median_height: null,
+      first_row_band: "unavailable",
+    } : tableProposalHeaderHints(run),
     truncation: {
       text_items: itemsTruncated ? "truncated" : "complete",
       ruled_rects: ruledTruncated ? "truncated" : "complete",

test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json

@@ -79,8 +79,8 @@
     {
       "role": "markdown_conversion_module",
       "path": "server/markdown-conversion.js",
-      "bytes": 187604,
-      "sha256": "befe5c77b455e0f02f593ec06186bd3b758e66dc09c8b468590d4710e14d4788"
+      "bytes": 192276,
+      "sha256": "838f9b1adb38faf5eb3b31d7be9479eb8521dff0fd7013dcd3c89fc3cbcb95d1"
     },
     {
       "role": "output_schemas_module",
@@ -131,7 +131,7 @@
       "sha256": "ef40501b2afbe4cd2adfa80480344783bbec1a00e8e9850086a5d044231d1f53"
     }
   ],
-  "validator_source_set_sha256": "afb4e11f1360d5732f977351c7787117e18c70aaf90f80ad02bb45572ee3ce59",
+  "validator_source_set_sha256": "6e875209853d97288e5eae1464c0c3a333f37b31ad3467839954aa4edbf657c4",
   "pdfjs_version": "5.4.624",
   "generation_contract": {
     "source_path": "source.pdf",

test/markdown-conversion.test.js

@@ -10,6 +10,7 @@ import {
   validatePdfLayoutSourceEvidence,
 } from "../server/layout-extraction.js";
 import {
+  analyzeValidatedPdfPagesForDocumentMap,
   renderPdfLayoutToMarkdown,
   validateMarkdownConversionSemantics,
 } from "../server/markdown-conversion.js";
@@ -465,6 +466,42 @@ describe("layout Markdown renderer", () => {
     expect(renderPdfLayoutToMarkdown(withBars, { allowHeaderlessTables })).toEqual(baseline);
   });
 
+  it("refines three-rule proposal sources without changing Markdown or promoting header roles", async () => {
+    const items = [
+      textItem("Table 1. Example", { top: 88, left: 50, fontSize: 10 }),
+      textItem("Concentration", { top: 113, left: 60, fontSize: 9 }),
+      textItem("level", { top: 124, left: 60, fontSize: 9 }),
+      textItem("A", { top: 120, left: 140, fontSize: 9, eol: false }),
+      textItem("B", { top: 120, left: 220, fontSize: 9 }),
+      ...["Control", "Low", "High"].flatMap((label, index) => [
+        textItem(label, { top: 145 + 15 * index, left: 60, fontSize: 9, eol: false }),
+        textItem(String(index), { top: 145 + 15 * index, left: 140, fontSize: 9, eol: false }),
+        textItem(String(index + 1), { top: 145 + 15 * index, left: 220, fontSize: 9 }),
+      ]),
+      textItem("3. Results", { top: 205, left: 50, fontSize: 10 }),
+    ];
+    const bars = [110, 140, 190].flatMap(y => [
+      { x: 50, y, width: 120, height: 1, verb: "fill" },
+      { x: 170, y, width: 100, height: 1, verb: "fill" },
+    ]);
+    const layout = attachRuledRects(await validatedSyntheticLayout([{ items }]), bars);
+    const baseline = renderPdfLayoutToMarkdown(layout);
+    const result = renderPdfLayoutToMarkdown(layout, { emitTableProposals: true });
+    expect(result.markdown).toBe(baseline.markdown);
+    expect(result.pages).toEqual(baseline.pages);
+    expect(result.gaps).toEqual(baseline.gaps);
+    const region = result.table_proposal_regions.find(value => value.bbox.y === 110);
+    expect(region).toBeDefined();
+    expect(region.text_items.map(item => item.text)).toEqual([
+      "Concentration", "level", "A", "B", "Control", "0", "1", "Low", "1", "2", "High", "2", "3",
+    ]);
+    expect(region.header_hints.status).toBe("unavailable");
+    const mapped = analyzeValidatedPdfPagesForDocumentMap(layout.pages);
+    expect(mapped[0].table_regions).toEqual(result.table_proposal_regions);
+    expect(analyzeValidatedPdfPagesForDocumentMap(layout.pages)).toEqual(mapped);
+    expect(() => validateMarkdownConversionSemantics(result, { layout })).not.toThrow();
+  });
+
   it("emits only inert data cells for an opted-in complete source-covered headerless grid", async () => {
     const layout = attachRuledRects(await validatedSyntheticLayout([{
       items: ruledGridItems({ headerSize: 12 }),