Retain thin filled PDF rule geometry without inferring cells

docs/MCP_CONTRACT.md

@@ -239,7 +239,17 @@ Stable item, line, and nonsemantic flow-block references support conservative
 reading order without claiming paragraphs or document structure. Version 1.2
 also retains bounded, axis-aligned solid-mask rectangle evidence with exact
 source-operation and transform provenance. Those rectangles are neutral paint
-evidence, not inferred rules or cells. Raster-only, mixed, hidden, clipped,
+evidence, not inferred rules or cells.
+The separate `ruled_rects` path also retains a single closed axis-aligned
+rectangle painted with `fill` or `eoFill` when its transformed shorter side is
+0.05 through 2 points and its longer side is at least 5 points. These thin
+filled bars retain their actual area and fill verb, not a synthesized stroke
+centerline, table cell or visibility claim. Compound paths, clipped-only and
+unpainted paths, strokes, and non-axis-aligned slivers do not qualify through
+this additional capture path. Existing rectangle caps and source replay apply.
+The Markdown cell-grid path keeps its previous 5-point minimum on both sides;
+retaining a thin bar does not establish missing row or column boundaries.
+Raster-only, mixed, hidden, clipped,
 duplicate, and OCR-overlay gaps remain explicit. The tool does not render, OCR,
 infer tables, or claim arbitrary schema extraction,
 and every item, character, or output limit is fail-closed with truncation

pdf-toolkit-mcp-share/server/layout-extraction.js

@@ -3125,7 +3125,8 @@ function rectangleSubpaths(commands) {
   return rectangles;
 }
 
-function transformedRectangle(segments, transform) {
+function transformedRectangle(segments, transform, minSize = RULED_RECT_MIN_SIZE,
+  axisTolerance = RULED_RECT_AXIS_TOLERANCE) {
   const transformedSegments = segments.map(([start, end]) => [
     applyViewportPoint(transform, start[0], start[1]),
     applyViewportPoint(transform, end[0], end[1]),
@@ -3135,8 +3136,8 @@ function transformedRectangle(segments, transform) {
   const axisAligned = transformedSegments.every(([start, end]) => {
     const dx = Math.abs(end[0] - start[0]);
     const dy = Math.abs(end[1] - start[1]);
-    return (dy <= RULED_RECT_AXIS_TOLERANCE && dx > RULED_RECT_AXIS_TOLERANCE)
-      || (dx <= RULED_RECT_AXIS_TOLERANCE && dy > RULED_RECT_AXIS_TOLERANCE);
+    return (dy <= axisTolerance && dx > axisTolerance)
+      || (dx <= axisTolerance && dy > axisTolerance);
   });
   if (!axisAligned) return null;
   const xs = points.map(point => point[0]);
@@ -3145,10 +3146,10 @@ function transformedRectangle(segments, transform) {
   const y = Math.min(...ys);
   const width = Math.max(...xs) - x;
   const height = Math.max(...ys) - y;
-  if (width < RULED_RECT_MIN_SIZE || height < RULED_RECT_MIN_SIZE) return null;
+  if (width < minSize || height < minSize) return null;
   const uniqueCorners = [];
   for (const point of points) {
-    if (!uniqueCorners.some(existing => Math.hypot(existing[0] - point[0], existing[1] - point[1]) <= RULED_RECT_AXIS_TOLERANCE)) {
+    if (!uniqueCorners.some(existing => Math.hypot(existing[0] - point[0], existing[1] - point[1]) <= axisTolerance)) {
       uniqueCorners.push(point);
     }
   }
@@ -3382,8 +3383,23 @@ function deriveOperatorEvidence(pdfjsLib, operators, viewportTransform) {
         }
       }
       if (verb !== null && buffer !== null) {
-        for (const segments of rectangleSubpaths(commands)) {
-          const rect = transformedRectangle(segments, combinedTransform);
+        const rectangles = rectangleSubpaths(commands);
+        for (const segments of rectangles) {
+          let rect = transformedRectangle(segments, combinedTransform);
+          // Producers also paint rules as thin filled rectangles, not stroked
+          // paths or image masks. Retain their actual area, never a fabricated
+          // centerline or a cell boundary. A single closed subpath avoids
+          // promoting holes/compound fill geometry into independent bars.
+          if (!rect && rectangles.length === 1
+            && [5, 6].includes(commands.length)
+            && commands[0].opcode === DRAW_OPS.moveTo
+            && commands.at(-1).opcode === DRAW_OPS.closePath
+            && commands.slice(1, -1).every(command => command.opcode === DRAW_OPS.lineTo)
+            && (paintOp === pdfjsLib.OPS?.fill || paintOp === pdfjsLib.OPS?.eoFill)) {
+            const bar = transformedRectangle(segments, combinedTransform, 0.05, 0.001);
+            if (bar && Math.min(bar.width, bar.height) <= 2
+              && Math.max(bar.width, bar.height) >= RULED_RECT_MIN_SIZE) rect = bar;
+          }
           if (rect) candidates.push({ ...rect, verb, operator_index: operatorIndex });
         }
       }

pdf-toolkit-mcp-share/server/markdown-conversion.js

@@ -1865,7 +1865,9 @@ function preprocessRuledRects(page) {
   if (!evidence || evidence.status !== "available") return null;
   const sourceRects = evidence.items
     .map((rect, order) => ({ ...rect, order }))
-    .filter(rect => rect.width > 0 && rect.height > 0);
+    // Thin filled rules are neutral area evidence, not individual table cells.
+    // Keep the established cell-candidate size floor independently of capture.
+    .filter(rect => rect.width >= 5 && rect.height >= 5);
   if (sourceRects.length === 0) return [];
 
   const medianWidth = median(sourceRects.map(rect => rect.width));

server/layout-extraction.js

@@ -3125,7 +3125,8 @@ function rectangleSubpaths(commands) {
   return rectangles;
 }
 
-function transformedRectangle(segments, transform) {
+function transformedRectangle(segments, transform, minSize = RULED_RECT_MIN_SIZE,
+  axisTolerance = RULED_RECT_AXIS_TOLERANCE) {
   const transformedSegments = segments.map(([start, end]) => [
     applyViewportPoint(transform, start[0], start[1]),
     applyViewportPoint(transform, end[0], end[1]),
@@ -3135,8 +3136,8 @@ function transformedRectangle(segments, transform) {
   const axisAligned = transformedSegments.every(([start, end]) => {
     const dx = Math.abs(end[0] - start[0]);
     const dy = Math.abs(end[1] - start[1]);
-    return (dy <= RULED_RECT_AXIS_TOLERANCE && dx > RULED_RECT_AXIS_TOLERANCE)
-      || (dx <= RULED_RECT_AXIS_TOLERANCE && dy > RULED_RECT_AXIS_TOLERANCE);
+    return (dy <= axisTolerance && dx > axisTolerance)
+      || (dx <= axisTolerance && dy > axisTolerance);
   });
   if (!axisAligned) return null;
   const xs = points.map(point => point[0]);
@@ -3145,10 +3146,10 @@ function transformedRectangle(segments, transform) {
   const y = Math.min(...ys);
   const width = Math.max(...xs) - x;
   const height = Math.max(...ys) - y;
-  if (width < RULED_RECT_MIN_SIZE || height < RULED_RECT_MIN_SIZE) return null;
+  if (width < minSize || height < minSize) return null;
   const uniqueCorners = [];
   for (const point of points) {
-    if (!uniqueCorners.some(existing => Math.hypot(existing[0] - point[0], existing[1] - point[1]) <= RULED_RECT_AXIS_TOLERANCE)) {
+    if (!uniqueCorners.some(existing => Math.hypot(existing[0] - point[0], existing[1] - point[1]) <= axisTolerance)) {
       uniqueCorners.push(point);
     }
   }
@@ -3382,8 +3383,23 @@ function deriveOperatorEvidence(pdfjsLib, operators, viewportTransform) {
         }
       }
       if (verb !== null && buffer !== null) {
-        for (const segments of rectangleSubpaths(commands)) {
-          const rect = transformedRectangle(segments, combinedTransform);
+        const rectangles = rectangleSubpaths(commands);
+        for (const segments of rectangles) {
+          let rect = transformedRectangle(segments, combinedTransform);
+          // Producers also paint rules as thin filled rectangles, not stroked
+          // paths or image masks. Retain their actual area, never a fabricated
+          // centerline or a cell boundary. A single closed subpath avoids
+          // promoting holes/compound fill geometry into independent bars.
+          if (!rect && rectangles.length === 1
+            && [5, 6].includes(commands.length)
+            && commands[0].opcode === DRAW_OPS.moveTo
+            && commands.at(-1).opcode === DRAW_OPS.closePath
+            && commands.slice(1, -1).every(command => command.opcode === DRAW_OPS.lineTo)
+            && (paintOp === pdfjsLib.OPS?.fill || paintOp === pdfjsLib.OPS?.eoFill)) {
+            const bar = transformedRectangle(segments, combinedTransform, 0.05, 0.001);
+            if (bar && Math.min(bar.width, bar.height) <= 2
+              && Math.max(bar.width, bar.height) >= RULED_RECT_MIN_SIZE) rect = bar;
+          }
           if (rect) candidates.push({ ...rect, verb, operator_index: operatorIndex });
         }
       }

server/markdown-conversion.js

@@ -1865,7 +1865,9 @@ function preprocessRuledRects(page) {
   if (!evidence || evidence.status !== "available") return null;
   const sourceRects = evidence.items
     .map((rect, order) => ({ ...rect, order }))
-    .filter(rect => rect.width > 0 && rect.height > 0);
+    // Thin filled rules are neutral area evidence, not individual table cells.
+    // Keep the established cell-candidate size floor independently of capture.
+    .filter(rect => rect.width >= 5 && rect.height >= 5);
   if (sourceRects.length === 0) return [];
 
   const medianWidth = median(sourceRects.map(rect => rect.width));

test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json

@@ -55,8 +55,8 @@
     {
       "role": "layout_extraction_module",
       "path": "server/layout-extraction.js",
-      "bytes": 277436,
-      "sha256": "40e559465467e7d12278d8281b14c48a1689573360111e8fed355f26b19ede75"
+      "bytes": 278406,
+      "sha256": "73448ebadf2fc8f6dad6048334fea90450bcfd9a9ab7d174c930e292e129834e"
     },
     {
       "role": "type3_cm_reference_module",
@@ -79,8 +79,8 @@
     {
       "role": "markdown_conversion_module",
       "path": "server/markdown-conversion.js",
-      "bytes": 187442,
-      "sha256": "59fe4915bc32fe734755d12ebec0df2d9829a907182e2a796fbb4c7309b5a538"
+      "bytes": 187604,
+      "sha256": "befe5c77b455e0f02f593ec06186bd3b758e66dc09c8b468590d4710e14d4788"
     },
     {
       "role": "output_schemas_module",
@@ -131,7 +131,7 @@
       "sha256": "ef40501b2afbe4cd2adfa80480344783bbec1a00e8e9850086a5d044231d1f53"
     }
   ],
-  "validator_source_set_sha256": "3df238fa35763b4461e65a1a884572d586646401415c32b9be98e413dc389a26",
+  "validator_source_set_sha256": "39715c5df6682ed5ecd398f6b3cfef18447a45f752bb500673cf2f6281402d39",
   "pdfjs_version": "5.4.624",
   "generation_contract": {
     "source_path": "source.pdf",

test/markdown-conversion.test.js

@@ -450,6 +450,21 @@ describe("layout Markdown renderer", () => {
     expect(baseline.gaps.some(gap => gap.message.includes("no source evidence distinguishes a header"))).toBe(true);
   });
 
+  it.each([false, true])("does not reinterpret retained thin filled bars as cell rectangles (opt-in=%s)", async allowHeaderlessTables => {
+    const layout = attachRuledRects(await validatedSyntheticLayout([{
+      items: ruledGridItems({ headerSize: 12 }),
+    }]), headerlessGridRects());
+    const baseline = renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables });
+    const withBars = structuredClone(layout);
+    withBars.pages[0].ruled_rects.items.push(
+      { x: 72, y: 120, width: 450, height: 0.4, verb: "fill" },
+      { x: 100, y: 112, width: 0.8, height: 192, verb: "fill" },
+    );
+    withBars.pages[0].ruled_rects.observed_count += 2;
+    withBars.pages[0].ruled_rects.returned_count += 2;
+    expect(renderPdfLayoutToMarkdown(withBars, { allowHeaderlessTables })).toEqual(baseline);
+  });
+
   it("emits only inert data cells for an opted-in complete source-covered headerless grid", async () => {
     const layout = attachRuledRects(await validatedSyntheticLayout([{
       items: ruledGridItems({ headerSize: 12 }),

test/read-pdf-layout.test.js

@@ -8,7 +8,7 @@ import { fileURLToPath } from "node:url";
 import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
 import { LoggingMessageNotificationSchema } from "@modelcontextprotocol/sdk/types.js";
-import { PDFDocument, PDFName, PDFNumber, StandardFonts, degrees } from "pdf-lib";
+import { PDFDocument, PDFName, PDFNumber, StandardFonts, degrees, rgb } from "pdf-lib";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
 import {
   extractPdfLayout,
@@ -1138,6 +1138,118 @@ describe("PDF.js factory directory contract", () => {
 });
 
 describe("Extraction IR v1.2.0 evidence blocks", () => {
+  it("retains thin filled bars as rectangles, not fabricated stroke segments", async () => {
+    const { result } = await runFake([fakeOperatorFixture([2, 2], [
+      rectPath(3, 10, 20, 200, 0.4), rectPath(16, 30, 40, 0.8, 100),
+    ])]);
+    expect(result.pages[0].ruled_rects.items).toEqual([
+      { x: 10, y: 771.6, width: 200, height: 0.4, verb: "fill" },
+      { x: 30, y: 652, width: 0.8, height: 100, verb: "fill" },
+    ]);
+    expect(result.pages[0].ruling_segments.items).toEqual([]);
+    expect(() => validatePdfLayoutSemantics(result)).not.toThrow();
+  });
+
+  it("measures the filled-bar gate after viewport and graphics transforms", async () => {
+    const { result } = await runFake([fakeOperatorFixture([10, 12, 2, 11], [
+      null, [0, 2, -2, 0, 100, 50], rectPath(3, 10, 20, 40, 0.4), null,
+    ])]);
+    expect(result.pages[0].ruled_rects.items).toEqual([
+      { x: 59.2, y: 642, width: 0.8, height: 80, verb: "fill" },
+    ]);
+    const thick = await runFake([fakeOperatorFixture([12, 2], [
+      [1, 0, 0, 3, 0, 0], rectPath(3, 10, 20, 100, 1),
+    ])]);
+    expect(thick.result.pages[0].ruled_rects.items).toEqual([]);
+  });
+
+  it("does not reinterpret clipped, unpainted, stroked, degenerate or thick slivers as filled bars", async () => {
+    const { result } = await runFake([fakeOperatorFixture(
+      [2, 2, 29, 2, 2, 2, 2, 2],
+      [rectPath(4, 10, 20, 100, 0.4), rectPath(17, 10, 30, 100, 0.4), null,
+        rectPath(21, 10, 40, 100, 0.4), rectPath(21, 10, 50, 100, 0.4),
+        rectPath(3, 10, 60, 100, 0.01), rectPath(3, 10, 70, 4, 1),
+        rectPath(3, 10, 80, 100, 3)],
+    )]);
+    expect(result.pages[0].ruled_rects.items).toEqual([]);
+  });
+
+  it("rejects compound, open, curved and sheared thin-fill candidates", async () => {
+    const compound = rectPath(16, 10, 20, 100, 1);
+    compound[1][0] = new Float32Array([
+      ...compound[1][0], ...rectPath(16, 20, 20.2, 80, 0.6)[1][0],
+    ]);
+    const open = rectPath(3, 10, 20, 100, 1);
+    open[1][0] = open[1][0].slice(0, -1);
+    const { result } = await runFake([fakeOperatorFixture([2, 2, 2, 12, 2], [
+      compound, open, curvePath(3), [1, 0.02, 0, 1, 0, 0], rectPath(3, 10, 20, 100, 1),
+    ])]);
+    expect(result.pages[0].ruled_rects.items).toEqual([]);
+  });
+
+  it.each([
+    [1, 150, 80],
+    [2, 0, 0, 200, 100, 110, 100],
+    [3, 0, 0, 110, 100],
+  ])("does not retain a closed rectangle followed by residual path commands %j", async (...tail) => {
+    const path = rectPath(3, 10, 20, 100, 1);
+    path[1][0] = new Float32Array([...path[1][0], ...tail]);
+    const { result } = await runFake([fakeOperatorFixture([2], [path])]);
+    expect(result.pages[0].ruled_rects.items).toEqual([]);
+  });
+
+  it("enforces the inclusive transformed thin-fill size boundaries", async () => {
+    const { result } = await runFake([fakeOperatorFixture([2, 2, 2, 2], [
+      // At a zero origin Float32 retains 0.05 just above the true minimum;
+      // storing 20.05 then subtracting 20 would instead fall below it.
+      rectPath(3, 10, 0, 5, 0.05), rectPath(3, 30, 40, 5, 2),
+      rectPath(3, 50, 60, 4.9, 1), rectPath(3, 70, 80, 5, 2.1),
+    ])]);
+    expect(result.pages[0].ruled_rects.items).toEqual([
+      { x: 10, y: 791.95, width: 5, height: 0.05, verb: "fill" },
+      { x: 30, y: 750, width: 5, height: 2, verb: "fill" },
+    ]);
+  });
+
+  it("keeps the existing rectangle cap and truthful omissions for thin filled bars", async () => {
+    const { result } = await runFake([fakeOperatorFixture(
+      Array.from({ length: 513 }, () => 2),
+      Array.from({ length: 513 }, (_, index) => rectPath(3, 10, 20 + index * 2, 100, 0.4)),
+    )]);
+    expect(result.pages[0].ruled_rects).toMatchObject({
+      status: "truncated", observed_count: 513, returned_count: 512,
+    });
+    expect(result.pages[0].errors).toContainEqual(expect.objectContaining({
+      stage: "ruled_rects", code: "RULED_RECT_PAGE_LIMIT",
+    }));
+  });
+
+  it("replays actual thin filled PDF rectangles and rejects omitted or substituted evidence", async () => {
+    const pdf = await PDFDocument.create();
+    const page = pdf.addPage([612, 792]);
+    page.drawRectangle({ x: 50, y: 700, width: 200, height: 0.4, color: rgb(0, 0, 0) });
+    page.drawRectangle({ x: 70, y: 600, width: 0.8, height: 60, color: rgb(0, 0, 0) });
+    const bytes = await pdf.save({ useObjectStreams: false });
+    const pdfjsLib = await import("pdfjs-dist/legacy/build/pdf.mjs");
+    const result = await extractPdfLayout({
+      pdfjsLib, pdfBytes: bytes, sourcePath: "/synthetic/thin-bars.pdf", sourceFileName: "thin-bars.pdf",
+      sourceSha256: createHash("sha256").update(bytes).digest("hex"),
+      requestedStartPage: 1, requestedEndPage: 1, maxItems: 1000,
+      maxCharacters: 50000, maxOutputCharacters: 200000, deadlineMs: 20000,
+    });
+    expect(result.pages[0].ruled_rects.items).toEqual([
+      { x: 50, y: 91.6, width: 200, height: 0.4, verb: "fill" },
+      { x: 70, y: 132, width: 0.8, height: 60, verb: "fill" },
+    ]);
+    await expect(validatePdfLayoutSourceEvidence(result, { pdfjsLib, sourceBytes: bytes })).resolves.toBe(result);
+    const omitted = structuredClone(result);
+    omitted.pages[0].ruled_rects = { status: "available", observed_count: 0, returned_count: 0, items: [] };
+    await expect(validatePdfLayoutSourceEvidence(omitted, { pdfjsLib, sourceBytes: bytes })).rejects.toThrow();
+    const changed = structuredClone(result);
+    changed.pages[0].ruled_rects.items[0].height = 0.8;
+    await expect(validatePdfLayoutSourceEvidence(changed, { pdfjsLib, sourceBytes: bytes })).rejects.toThrow();
+  });
+
   it("captures only normalized stroke-bearing axis-aligned ruling segments with typed cap accounting", async () => {
     const basic = await runFake([fakeOperatorFixture(
       [2, 2, 2, 2, 2],