From 3d3193e662f93d78baadc3ca0f470e1958d930d7 Mon Sep 17 00:00:00 2001 From: silverstein Date: Wed, 21 Oct 2026 04:00:04 +0000 Subject: [PATCH 1/5] Bound abandoned table proposals by captioned three-rule source envelopes --- .../server/markdown-conversion.js | 86 ++++++++++++++-- server/markdown-conversion.js | 86 ++++++++++++++-- test/table-proposal-envelope.test.js | 97 +++++++++++++++++++ 3 files changed, 253 insertions(+), 16 deletions(-) create mode 100644 test/table-proposal-envelope.test.js diff --git a/pdf-toolkit-mcp-share/server/markdown-conversion.js b/pdf-toolkit-mcp-share/server/markdown-conversion.js index b4ea266..f6bc176 100644 --- a/pdf-toolkit-mcp-share/server/markdown-conversion.js +++ b/pdf-toolkit-mcp-share/server/markdown-conversion.js @@ -2815,15 +2815,82 @@ function tableProposalHeaderHints(run) { }; } +function threeRuleProposalEnvelope(page, run) { + // This refines an already abandoned proposal, never reconstructs a table. + // Three full-width paint bands and a nearby caption can bound the source + // region without establishing columns, cell cuts, or header semantics. + const evidence = page.ruled_rects; + if (evidence?.status !== "available" || page.truncation?.truncated + || evidence.items.length > MAX_TABLE_PROPOSAL_RULED_RECTS + || evidence.observed_count !== evidence.items.length + || evidence.returned_count !== evidence.items.length) return null; + const text = page.raw_items.filter(item => item.text_kind === "non_whitespace"); + if (text.some(item => !item.geometry_valid || !item.bbox)) return null; + const tolerance = 0.05; + const bars = evidence.items.filter(rect => ["fill", "eoFill"].includes(rect.verb) + && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5) + .sort((a, b) => a.y - b.y || a.x - b.x); + const bands = []; + for (const bar of bars) { + const prior = bands.at(-1); + if (prior && Math.abs(prior.y - bar.y) <= tolerance + && Math.abs(prior.height - bar.height) <= tolerance + && Math.abs(prior.x + prior.width - bar.x) <= tolerance) { + prior.width = bar.x + bar.width - prior.x; + } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height }); + } + const contains = (outer, inner) => inner.x >= outer.x && inner.y >= outer.y + && inner.x + inner.width <= outer.x + outer.width + && inner.y + inner.height <= outer.y + outer.height; + const runIds = new Set(run.map(row => row.line.id)); + const candidates = []; + for (const top of bands) { + if (top.width < 80) continue; + const aligned = bands.filter(band => Math.abs(band.x - top.x) <= tolerance + && Math.abs(band.width - top.width) <= tolerance); + if (aligned.length !== 3 || aligned[0] !== top) continue; + const [, middle, bottom] = aligned; + if (middle.y - top.y - top.height < 5 || bottom.y - middle.y - middle.height < 5 + || bottom.y + bottom.height - top.y > 300) continue; + const bbox = { x: top.x, y: top.y, width: top.width, + height: bottom.y + bottom.height - top.y }; + const interior = { x: bbox.x, y: top.y + top.height, width: bbox.width, + height: bottom.y - top.y - top.height }; + // Reject partial/competing paint inside this envelope, not just extra + // full-width rules. Clip boxes are retained as evidence, not paint bands. + if (bands.some(band => !aligned.includes(band) && rectsOverlap(band, bbox))) continue; + const captions = page.lines.filter(line => /^Table\s+\d+\b/iu.test(line.text.trim()) + && line.x >= bbox.x && line.x + line.width <= bbox.x + bbox.width + && line.y + line.height <= top.y + && top.y - line.y - line.height <= Math.min(48, 3 * line.height)); + if (captions.length !== 1) continue; + const intersecting = text.filter(item => rectsOverlap(item.bbox, bbox)); + if (intersecting.length === 0 || intersecting.some(item => !contains(interior, item.bbox) + || rectsOverlap(item.bbox, middle))) continue; + const lines = page.lines.filter(line => rectsOverlap(line, interior)); + if (lines.some(line => !contains(interior, line))) continue; + // Reopen every retained source item, including lines missed by the old + // adjacent-run heuristic. Never silently drop an unassigned source item. + const itemIds = lines.flatMap(line => line.item_ids) + .filter(id => intersecting.some(item => item.id === id)); + if (new Set(itemIds).size !== itemIds.length || itemIds.length !== intersecting.length) continue; + if (lines.filter(line => runIds.has(line.id)).length < 2 + || !lines.some(line => line.y + line.height <= middle.y) + || !lines.some(line => line.y >= middle.y + middle.height)) continue; + const itemById = new Map(page.raw_items.map(item => [item.id, item])); + candidates.push({ bbox, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) }); + } + return candidates.length === 1 ? candidates[0] : null; +} + /** - * Build one deterministic, bounded proposal-region descriptor for a table run - * the renderer abandoned. Pure over the page IR and the run; no token, no I/O, - * no time/RNG. The proposal_token that binds it to (source sha, IR version, - * region_id) is added by the handler, which owns the source identity. Exported - * for the bounded-truncation regression, which needs to force an over-cap - * region without synthesizing a full validated IR. + * Build a bounded proposal from an abandoned run. A uniquely captioned + * three-rule envelope can refine source membership but supplies no topology + * or header authority. Pure over source-validated IR; no token, time or I/O. */ export function buildTableProposalRegion(page, run, reason, ordinal) { + const envelope = threeRuleProposalEnvelope(page, run); + if (envelope) run = envelope.run; const itemById = new Map(page.raw_items.map(item => [item.id, item])); let minX = Infinity; let minY = Infinity; @@ -2835,7 +2902,7 @@ export function buildTableProposalRegion(page, run, reason, ordinal) { maxX = Math.max(maxX, line.x + line.width); maxY = Math.max(maxY, line.y + line.height); } - const bbox = { x: minX, y: minY, width: maxX - minX, height: maxY - minY }; + const bbox = envelope?.bbox ?? { x: minX, y: minY, width: maxX - minX, height: maxY - minY }; const allItems = []; for (const { line } of run) { @@ -2895,7 +2962,10 @@ export function buildTableProposalRegion(page, run, reason, ordinal) { ruled_rects: ruledRects, ruling_segments: rulingSegments, painted_rectangles: paintedRectangles, - header_hints: tableProposalHeaderHints(run), + header_hints: envelope ? { + status: "unavailable", first_row_height: null, body_median_height: null, + first_row_band: "unavailable", + } : tableProposalHeaderHints(run), truncation: { text_items: itemsTruncated ? "truncated" : "complete", ruled_rects: ruledTruncated ? "truncated" : "complete", diff --git a/server/markdown-conversion.js b/server/markdown-conversion.js index b4ea266..f6bc176 100644 --- a/server/markdown-conversion.js +++ b/server/markdown-conversion.js @@ -2815,15 +2815,82 @@ function tableProposalHeaderHints(run) { }; } +function threeRuleProposalEnvelope(page, run) { + // This refines an already abandoned proposal, never reconstructs a table. + // Three full-width paint bands and a nearby caption can bound the source + // region without establishing columns, cell cuts, or header semantics. + const evidence = page.ruled_rects; + if (evidence?.status !== "available" || page.truncation?.truncated + || evidence.items.length > MAX_TABLE_PROPOSAL_RULED_RECTS + || evidence.observed_count !== evidence.items.length + || evidence.returned_count !== evidence.items.length) return null; + const text = page.raw_items.filter(item => item.text_kind === "non_whitespace"); + if (text.some(item => !item.geometry_valid || !item.bbox)) return null; + const tolerance = 0.05; + const bars = evidence.items.filter(rect => ["fill", "eoFill"].includes(rect.verb) + && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5) + .sort((a, b) => a.y - b.y || a.x - b.x); + const bands = []; + for (const bar of bars) { + const prior = bands.at(-1); + if (prior && Math.abs(prior.y - bar.y) <= tolerance + && Math.abs(prior.height - bar.height) <= tolerance + && Math.abs(prior.x + prior.width - bar.x) <= tolerance) { + prior.width = bar.x + bar.width - prior.x; + } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height }); + } + const contains = (outer, inner) => inner.x >= outer.x && inner.y >= outer.y + && inner.x + inner.width <= outer.x + outer.width + && inner.y + inner.height <= outer.y + outer.height; + const runIds = new Set(run.map(row => row.line.id)); + const candidates = []; + for (const top of bands) { + if (top.width < 80) continue; + const aligned = bands.filter(band => Math.abs(band.x - top.x) <= tolerance + && Math.abs(band.width - top.width) <= tolerance); + if (aligned.length !== 3 || aligned[0] !== top) continue; + const [, middle, bottom] = aligned; + if (middle.y - top.y - top.height < 5 || bottom.y - middle.y - middle.height < 5 + || bottom.y + bottom.height - top.y > 300) continue; + const bbox = { x: top.x, y: top.y, width: top.width, + height: bottom.y + bottom.height - top.y }; + const interior = { x: bbox.x, y: top.y + top.height, width: bbox.width, + height: bottom.y - top.y - top.height }; + // Reject partial/competing paint inside this envelope, not just extra + // full-width rules. Clip boxes are retained as evidence, not paint bands. + if (bands.some(band => !aligned.includes(band) && rectsOverlap(band, bbox))) continue; + const captions = page.lines.filter(line => /^Table\s+\d+\b/iu.test(line.text.trim()) + && line.x >= bbox.x && line.x + line.width <= bbox.x + bbox.width + && line.y + line.height <= top.y + && top.y - line.y - line.height <= Math.min(48, 3 * line.height)); + if (captions.length !== 1) continue; + const intersecting = text.filter(item => rectsOverlap(item.bbox, bbox)); + if (intersecting.length === 0 || intersecting.some(item => !contains(interior, item.bbox) + || rectsOverlap(item.bbox, middle))) continue; + const lines = page.lines.filter(line => rectsOverlap(line, interior)); + if (lines.some(line => !contains(interior, line))) continue; + // Reopen every retained source item, including lines missed by the old + // adjacent-run heuristic. Never silently drop an unassigned source item. + const itemIds = lines.flatMap(line => line.item_ids) + .filter(id => intersecting.some(item => item.id === id)); + if (new Set(itemIds).size !== itemIds.length || itemIds.length !== intersecting.length) continue; + if (lines.filter(line => runIds.has(line.id)).length < 2 + || !lines.some(line => line.y + line.height <= middle.y) + || !lines.some(line => line.y >= middle.y + middle.height)) continue; + const itemById = new Map(page.raw_items.map(item => [item.id, item])); + candidates.push({ bbox, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) }); + } + return candidates.length === 1 ? candidates[0] : null; +} + /** - * Build one deterministic, bounded proposal-region descriptor for a table run - * the renderer abandoned. Pure over the page IR and the run; no token, no I/O, - * no time/RNG. The proposal_token that binds it to (source sha, IR version, - * region_id) is added by the handler, which owns the source identity. Exported - * for the bounded-truncation regression, which needs to force an over-cap - * region without synthesizing a full validated IR. + * Build a bounded proposal from an abandoned run. A uniquely captioned + * three-rule envelope can refine source membership but supplies no topology + * or header authority. Pure over source-validated IR; no token, time or I/O. */ export function buildTableProposalRegion(page, run, reason, ordinal) { + const envelope = threeRuleProposalEnvelope(page, run); + if (envelope) run = envelope.run; const itemById = new Map(page.raw_items.map(item => [item.id, item])); let minX = Infinity; let minY = Infinity; @@ -2835,7 +2902,7 @@ export function buildTableProposalRegion(page, run, reason, ordinal) { maxX = Math.max(maxX, line.x + line.width); maxY = Math.max(maxY, line.y + line.height); } - const bbox = { x: minX, y: minY, width: maxX - minX, height: maxY - minY }; + const bbox = envelope?.bbox ?? { x: minX, y: minY, width: maxX - minX, height: maxY - minY }; const allItems = []; for (const { line } of run) { @@ -2895,7 +2962,10 @@ export function buildTableProposalRegion(page, run, reason, ordinal) { ruled_rects: ruledRects, ruling_segments: rulingSegments, painted_rectangles: paintedRectangles, - header_hints: tableProposalHeaderHints(run), + header_hints: envelope ? { + status: "unavailable", first_row_height: null, body_median_height: null, + first_row_band: "unavailable", + } : tableProposalHeaderHints(run), truncation: { text_items: itemsTruncated ? "truncated" : "complete", ruled_rects: ruledTruncated ? "truncated" : "complete", diff --git a/test/table-proposal-envelope.test.js b/test/table-proposal-envelope.test.js new file mode 100644 index 0000000..e855729 --- /dev/null +++ b/test/table-proposal-envelope.test.js @@ -0,0 +1,97 @@ +import { describe, expect, it } from "vitest"; +import { buildTableProposalRegion } from "../server/markdown-conversion.js"; + +function fixture() { + const rows = [ + ["caption", "Table 1. Example values", 90, 60, 180], + ["header-1", "Concentration", 112, 60, 60], + ["header-2", "level", 124, 70, 30], + ["columns", "A B C", 118, 150, 100], + ["body-1", "Control 0 0 0", 145, 60, 190], + ["body-2", "Low 1 2 3", 160, 60, 190], + ["body-3", "High 4 5 6", 175, 60, 190], + ["results", "Results", 205, 50, 50], + ["prose", "Ordinary following prose", 220, 50, 200], + ]; + const raw_items = rows.map(([id, text, y, x, width], index) => ({ + id, text, reading_order_index: index, line_id: id + "-line", column_index: 0, + geometry_valid: true, text_kind: "non_whitespace", + bbox: { x, y, width, height: 10 }, raw_transform: [], quad: null, + })); + const lines = raw_items.map(item => ({ ...item.bbox, id: item.line_id, + text: item.text, item_ids: [item.id] })); + const items = [110, 140, 190].flatMap(y => [ + { x: 50, y, width: 120, height: 1, verb: "fill" }, + { x: 170, y, width: 100, height: 1, verb: "fill" }, + ]); + const page = { page: 1, raw_items, lines, truncation: { truncated: false }, + ruled_rects: { status: "available", observed_count: 6, returned_count: 6, items }, + ruling_segments: { status: "available", truncated: false, items: [] }, + painted_rectangles: { items: [] } }; + const run = lines.slice(3).map(line => ({ line, + cells: raw_items.filter(item => item.line_id === line.id) })); + return { page, run }; +} + +function packet({ page, run }) { + return buildTableProposalRegion(page, run, "topology", 2); +} + +function oldPacket(input) { + const copy = structuredClone(input); + copy.page.ruled_rects.status = "unavailable"; + return packet(copy); +} + +describe("abandoned three-rule table source envelope", () => { + it("retains both header lines and removes only outside prose without asserting topology", () => { + const input = fixture(); + const before = structuredClone(input); + const result = packet(input); + expect(result.text_items.map(item => item.id)).toEqual([ + "header-1", "header-2", "columns", "body-1", "body-2", "body-3", + ]); + expect(result.bbox).toEqual({ x: 50, y: 110, width: 220, height: 81 }); + expect(result.ruled_rects).toHaveLength(6); + expect(result.region_id).toBe("p1-t2"); + expect(result.reason).toBe("TABLE_TOPOLOGY_UNKNOWN"); + expect(result.header_hints).toEqual({ status: "unavailable", first_row_height: null, + body_median_height: null, first_row_band: "unavailable" }); + expect(result).not.toHaveProperty("cells"); + expect(packet(input)).toEqual(result); + expect(input).toEqual(before); + }); + + const attacks = { + no_caption: input => { input.page.lines[0].text = "Section 1. Example values"; }, + distant_caption: input => { input.page.lines[0].y = 10; }, + duplicate_caption: input => { input.page.lines.push({ ...input.page.lines[0], id: "other-caption" }); }, + disconnected_band: input => { input.page.ruled_rects.items[1].x += 3; }, + overlapping_band: input => { input.page.ruled_rects.items[1].x -= 3; }, + unequal_width: input => { input.page.ruled_rects.items[5].width -= 10; }, + missing_band: input => { input.page.ruled_rects.items.splice(2, 2); }, + fourth_band: input => { input.page.ruled_rects.items.push({ x: 50, y: 155, width: 220, height: 1, verb: "fill" }); }, + interior_fraction_bar: input => { input.page.ruled_rects.items.push({ x: 65, y: 155, width: 20, height: 1, verb: "fill" }); }, + clip_only: input => { input.page.ruled_rects.items.forEach(item => { item.verb = "clip"; }); }, + truncated_geometry: input => { input.page.ruled_rects.status = "truncated"; }, + truncated_text: input => { input.page.truncation.truncated = true; }, + missing_geometry: input => { input.page.raw_items[1].geometry_valid = false; }, + straddling_outer_boundary: input => { input.page.raw_items[1].bbox.y = 109; }, + straddling_middle_rule: input => { input.page.raw_items[1].bbox.y = 135; }, + straddling_line: input => { input.page.lines[1].x = 40; }, + unassigned_item: input => { input.page.lines[1].item_ids = []; }, + duplicate_membership: input => { input.page.lines[1].item_ids.push("header-1"); }, + unrelated_run: input => { input.run = input.run.slice(-2); }, + }; + it.each(Object.keys(attacks))("preserves the original proposal for %s", name => { + const input = fixture(); + attacks[name](input); + input.page.ruled_rects.observed_count = input.page.ruled_rects.items.length; + input.page.ruled_rects.returned_count = input.page.ruled_rects.items.length; + const actual = packet(input); + const old = oldPacket(input); + expect(actual.bbox).toEqual(old.bbox); + expect(actual.text_items).toEqual(old.text_items); + expect(actual.header_hints).toEqual(old.header_hints); + }); +}); From eb18866a91024500d19aa59e5bc70a308ab31b4d Mon Sep 17 00:00:00 2001 From: silverstein Date: Wed, 21 Oct 2026 04:00:59 +0000 Subject: [PATCH 2/5] Cover proposal envelope in renderer and document-map replay --- test/markdown-conversion.test.js | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/markdown-conversion.test.js b/test/markdown-conversion.test.js index 91e8042..c1a397b 100644 --- a/test/markdown-conversion.test.js +++ b/test/markdown-conversion.test.js @@ -10,6 +10,7 @@ import { validatePdfLayoutSourceEvidence, } from "../server/layout-extraction.js"; import { + analyzeValidatedPdfPagesForDocumentMap, renderPdfLayoutToMarkdown, validateMarkdownConversionSemantics, } from "../server/markdown-conversion.js"; @@ -465,6 +466,42 @@ describe("layout Markdown renderer", () => { expect(renderPdfLayoutToMarkdown(withBars, { allowHeaderlessTables })).toEqual(baseline); }); + it("refines three-rule proposal sources without changing Markdown or promoting header roles", async () => { + const items = [ + textItem("Table 1. Example", { top: 88, left: 50, fontSize: 10 }), + textItem("Concentration", { top: 113, left: 60, fontSize: 9 }), + textItem("level", { top: 124, left: 60, fontSize: 9 }), + textItem("A", { top: 120, left: 140, fontSize: 9, eol: false }), + textItem("B", { top: 120, left: 220, fontSize: 9 }), + ...["Control", "Low", "High"].flatMap((label, index) => [ + textItem(label, { top: 145 + 15 * index, left: 60, fontSize: 9, eol: false }), + textItem(String(index), { top: 145 + 15 * index, left: 140, fontSize: 9, eol: false }), + textItem(String(index + 1), { top: 145 + 15 * index, left: 220, fontSize: 9 }), + ]), + textItem("3. Results", { top: 205, left: 50, fontSize: 10 }), + ]; + const bars = [110, 140, 190].flatMap(y => [ + { x: 50, y, width: 120, height: 1, verb: "fill" }, + { x: 170, y, width: 100, height: 1, verb: "fill" }, + ]); + const layout = attachRuledRects(await validatedSyntheticLayout([{ items }]), bars); + const baseline = renderPdfLayoutToMarkdown(layout); + const result = renderPdfLayoutToMarkdown(layout, { emitTableProposals: true }); + expect(result.markdown).toBe(baseline.markdown); + expect(result.pages).toEqual(baseline.pages); + expect(result.gaps).toEqual(baseline.gaps); + const region = result.table_proposal_regions.find(value => value.bbox.y === 110); + expect(region).toBeDefined(); + expect(region.text_items.map(item => item.text)).toEqual([ + "Concentration", "level", "A", "B", "Control", "0", "1", "Low", "1", "2", "High", "2", "3", + ]); + expect(region.header_hints.status).toBe("unavailable"); + const mapped = analyzeValidatedPdfPagesForDocumentMap(layout.pages); + expect(mapped[0].table_regions).toEqual(result.table_proposal_regions); + expect(analyzeValidatedPdfPagesForDocumentMap(layout.pages)).toEqual(mapped); + expect(() => validateMarkdownConversionSemantics(result, { layout })).not.toThrow(); + }); + it("emits only inert data cells for an opted-in complete source-covered headerless grid", async () => { const layout = attachRuledRects(await validatedSyntheticLayout([{ items: ruledGridItems({ headerSize: 12 }), From cb11b97956332e26735f3a797ca4146e5a7e8f73 Mon Sep 17 00:00:00 2001 From: silverstein Date: Wed, 21 Oct 2026 04:02:13 +0000 Subject: [PATCH 3/5] Reject competing or incomplete paint evidence in proposal envelopes --- .../server/markdown-conversion.js | 14 ++++++++++++-- server/markdown-conversion.js | 14 ++++++++++++-- test/table-proposal-envelope.test.js | 17 +++++++++++++++-- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/pdf-toolkit-mcp-share/server/markdown-conversion.js b/pdf-toolkit-mcp-share/server/markdown-conversion.js index f6bc176..324092d 100644 --- a/pdf-toolkit-mcp-share/server/markdown-conversion.js +++ b/pdf-toolkit-mcp-share/server/markdown-conversion.js @@ -2824,6 +2824,11 @@ function threeRuleProposalEnvelope(page, run) { || evidence.items.length > MAX_TABLE_PROPOSAL_RULED_RECTS || evidence.observed_count !== evidence.items.length || evidence.returned_count !== evidence.items.length) return null; + if ([page.ruling_segments, page.painted_rectangles].some(channel => ( + channel?.status !== "available" || channel.truncated + || channel.observed_count !== channel.items.length + || channel.returned_count !== channel.items.length + ))) return null; const text = page.raw_items.filter(item => item.text_kind === "non_whitespace"); if (text.some(item => !item.geometry_valid || !item.bbox)) return null; const tolerance = 0.05; @@ -2837,7 +2842,8 @@ function threeRuleProposalEnvelope(page, run) { && Math.abs(prior.height - bar.height) <= tolerance && Math.abs(prior.x + prior.width - bar.x) <= tolerance) { prior.width = bar.x + bar.width - prior.x; - } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height }); + prior.parts.push(bar); + } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] }); } const contains = (outer, inner) => inner.x >= outer.x && inner.y >= outer.y && inner.x + inner.width <= outer.x + outer.width @@ -2858,7 +2864,11 @@ function threeRuleProposalEnvelope(page, run) { height: bottom.y - top.y - top.height }; // Reject partial/competing paint inside this envelope, not just extra // full-width rules. Clip boxes are retained as evidence, not paint bands. - if (bands.some(band => !aligned.includes(band) && rectsOverlap(band, bbox))) continue; + const constituents = new Set(aligned.flatMap(band => band.parts)); + if (evidence.items.some(rect => !["clip", "eoClip"].includes(rect.verb) + && !constituents.has(rect) && rectsOverlap(rect, bbox)) + || page.ruling_segments.items.some(segment => rulingSegmentOverlapsRegion(segment, bbox)) + || page.painted_rectangles.items.some(rect => rectsOverlap(rect.bbox, bbox))) continue; const captions = page.lines.filter(line => /^Table\s+\d+\b/iu.test(line.text.trim()) && line.x >= bbox.x && line.x + line.width <= bbox.x + bbox.width && line.y + line.height <= top.y diff --git a/server/markdown-conversion.js b/server/markdown-conversion.js index f6bc176..324092d 100644 --- a/server/markdown-conversion.js +++ b/server/markdown-conversion.js @@ -2824,6 +2824,11 @@ function threeRuleProposalEnvelope(page, run) { || evidence.items.length > MAX_TABLE_PROPOSAL_RULED_RECTS || evidence.observed_count !== evidence.items.length || evidence.returned_count !== evidence.items.length) return null; + if ([page.ruling_segments, page.painted_rectangles].some(channel => ( + channel?.status !== "available" || channel.truncated + || channel.observed_count !== channel.items.length + || channel.returned_count !== channel.items.length + ))) return null; const text = page.raw_items.filter(item => item.text_kind === "non_whitespace"); if (text.some(item => !item.geometry_valid || !item.bbox)) return null; const tolerance = 0.05; @@ -2837,7 +2842,8 @@ function threeRuleProposalEnvelope(page, run) { && Math.abs(prior.height - bar.height) <= tolerance && Math.abs(prior.x + prior.width - bar.x) <= tolerance) { prior.width = bar.x + bar.width - prior.x; - } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height }); + prior.parts.push(bar); + } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] }); } const contains = (outer, inner) => inner.x >= outer.x && inner.y >= outer.y && inner.x + inner.width <= outer.x + outer.width @@ -2858,7 +2864,11 @@ function threeRuleProposalEnvelope(page, run) { height: bottom.y - top.y - top.height }; // Reject partial/competing paint inside this envelope, not just extra // full-width rules. Clip boxes are retained as evidence, not paint bands. - if (bands.some(band => !aligned.includes(band) && rectsOverlap(band, bbox))) continue; + const constituents = new Set(aligned.flatMap(band => band.parts)); + if (evidence.items.some(rect => !["clip", "eoClip"].includes(rect.verb) + && !constituents.has(rect) && rectsOverlap(rect, bbox)) + || page.ruling_segments.items.some(segment => rulingSegmentOverlapsRegion(segment, bbox)) + || page.painted_rectangles.items.some(rect => rectsOverlap(rect.bbox, bbox))) continue; const captions = page.lines.filter(line => /^Table\s+\d+\b/iu.test(line.text.trim()) && line.x >= bbox.x && line.x + line.width <= bbox.x + bbox.width && line.y + line.height <= top.y diff --git a/test/table-proposal-envelope.test.js b/test/table-proposal-envelope.test.js index e855729..443094e 100644 --- a/test/table-proposal-envelope.test.js +++ b/test/table-proposal-envelope.test.js @@ -26,8 +26,8 @@ function fixture() { ]); const page = { page: 1, raw_items, lines, truncation: { truncated: false }, ruled_rects: { status: "available", observed_count: 6, returned_count: 6, items }, - ruling_segments: { status: "available", truncated: false, items: [] }, - painted_rectangles: { items: [] } }; + ruling_segments: { status: "available", truncated: false, observed_count: 0, returned_count: 0, items: [] }, + painted_rectangles: { status: "available", truncated: false, observed_count: 0, returned_count: 0, items: [] } }; const run = lines.slice(3).map(line => ({ line, cells: raw_items.filter(item => item.line_id === line.id) })); return { page, run }; @@ -72,6 +72,16 @@ describe("abandoned three-rule table source envelope", () => { missing_band: input => { input.page.ruled_rects.items.splice(2, 2); }, fourth_band: input => { input.page.ruled_rects.items.push({ x: 50, y: 155, width: 220, height: 1, verb: "fill" }); }, interior_fraction_bar: input => { input.page.ruled_rects.items.push({ x: 65, y: 155, width: 20, height: 1, verb: "fill" }); }, + thick_foreign_fill: input => { input.page.ruled_rects.items.push({ x: 50, y: 155, width: 220, height: 3, verb: "fill" }); }, + short_foreign_fill: input => { input.page.ruled_rects.items.push({ x: 65, y: 155, width: 4, height: 1, verb: "fill" }); }, + foreign_stroke: input => { + input.page.ruling_segments.items.push({ orientation: "horizontal", x1: 50, x2: 270, y1: 155, y2: 155 }); + }, + foreign_paint: input => { input.page.painted_rectangles.items.push({ bbox: { x: 50, y: 155, width: 220, height: 3 } }); }, + unavailable_strokes: input => { input.page.ruling_segments.status = "unavailable"; }, + truncated_strokes: input => { input.page.ruling_segments.truncated = true; }, + unavailable_paint: input => { input.page.painted_rectangles.status = "unavailable"; }, + truncated_paint: input => { input.page.painted_rectangles.truncated = true; }, clip_only: input => { input.page.ruled_rects.items.forEach(item => { item.verb = "clip"; }); }, truncated_geometry: input => { input.page.ruled_rects.status = "truncated"; }, truncated_text: input => { input.page.truncation.truncated = true; }, @@ -88,6 +98,9 @@ describe("abandoned three-rule table source envelope", () => { attacks[name](input); input.page.ruled_rects.observed_count = input.page.ruled_rects.items.length; input.page.ruled_rects.returned_count = input.page.ruled_rects.items.length; + for (const channel of [input.page.ruling_segments, input.page.painted_rectangles]) { + channel.observed_count = channel.returned_count = channel.items.length; + } const actual = packet(input); const old = oldPacket(input); expect(actual.bbox).toEqual(old.bbox); From 63c5b086d18ed4460ec03f1e86a3266d661cdc25 Mon Sep 17 00:00:00 2001 From: silverstein Date: Wed, 21 Oct 2026 04:03:37 +0000 Subject: [PATCH 4/5] Reject stale proposal membership after envelope replay --- test/table-proposal-envelope.test.js | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/table-proposal-envelope.test.js b/test/table-proposal-envelope.test.js index 443094e..b66e8e2 100644 --- a/test/table-proposal-envelope.test.js +++ b/test/table-proposal-envelope.test.js @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { buildTableProposalRegion } from "../server/markdown-conversion.js"; +import { verifyTableProposalAgainstRegion } from "../server/table-proposal-verification.js"; function fixture() { const rows = [ @@ -62,6 +63,28 @@ describe("abandoned three-rule table source envelope", () => { expect(input).toEqual(before); }); + it("rejects old item assignments against freshly rebuilt region membership", () => { + const input = fixture(); + const region = packet(input); + const old = oldPacket(input); + const result = verifyTableProposalAgainstRegion({ + source: { file_name: "synthetic.pdf", sha256: "b".repeat(64), size_bytes: 123 }, + layout: { name: "pdf-tools.extraction-ir", version: "1.6.0", + parser_name: "pdfjs-dist", parser_version: "5.4.624" }, + regionId: region.region_id, page: 1, region, + // Tokens bind source/IR/region, not membership. A matching old token + // must not make missing header items or outside-prose assignments valid. + proposalToken: "a".repeat(64), expectedProposalToken: "a".repeat(64), + cells: old.text_items.map((item, index) => ({ row: index, column: 0, + rowspan: 1, colspan: 1, item_ids: [item.id] })), + }); + expect(result.status).toBe("rejected"); + expect(result.reason_codes).toContain("TABLE_PROPOSAL_ITEM_UNKNOWN"); + expect(result.reason_codes).toContain("TABLE_PROPOSAL_ITEM_MISSING"); + expect(result.reason_codes).toContain("TABLE_PROPOSAL_HEADER_UNSUPPORTED"); + expect(result.table).toBeNull(); + }); + const attacks = { no_caption: input => { input.page.lines[0].text = "Section 1. Example values"; }, distant_caption: input => { input.page.lines[0].y = 10; }, From 35ef51d78567b90f97da9a1e64ac1d02dffd76b9 Mon Sep 17 00:00:00 2001 From: silverstein Date: Wed, 21 Oct 2026 04:05:12 +0000 Subject: [PATCH 5/5] Document proposal-only boundary and refresh source evidence --- docs/MAINTAINERS.md | 10 ++++++++++ .../extraction/phase1/layout-occurrence-oracle.v1.json | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/MAINTAINERS.md b/docs/MAINTAINERS.md index 8ef90e5..0c97446 100644 --- a/docs/MAINTAINERS.md +++ b/docs/MAINTAINERS.md @@ -303,6 +303,16 @@ structured spans are authoritative. Never describe an accepted result as proof of the one uniquely correct or semantically correct topology. Rejections and ordinary conversions remain fail-closed and emit no inferred table. +Proposal regions may use one uniquely captioned three-horizontal-fill-band +envelope to recover complete source membership, including multiline heading +text, while excluding nearby prose. This is region selection only: it does not +infer columns, cell boundaries or header roles. All intersecting text must be +accounted for; competing paint, incomplete geometry, ambiguous captions and +boundary-straddling items preserve the original unrefined proposal. Refined +regions suppress header hints. The same source region is used by document maps +and fresh proposal verification. Ordinary Markdown and conversion gaps do not +change, and matching source/region tokens do not excuse obsolete assignments. + Filesystem operations and rasterization happen locally, and PDF Tools does not upload files to a separate PDF service. Text, images, and metadata returned through MCP may still be processed under the selected host or model provider's diff --git a/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json b/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json index 8f9789b..10b81b3 100644 --- a/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json +++ b/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json @@ -79,8 +79,8 @@ { "role": "markdown_conversion_module", "path": "server/markdown-conversion.js", - "bytes": 187604, - "sha256": "befe5c77b455e0f02f593ec06186bd3b758e66dc09c8b468590d4710e14d4788" + "bytes": 192276, + "sha256": "838f9b1adb38faf5eb3b31d7be9479eb8521dff0fd7013dcd3c89fc3cbcb95d1" }, { "role": "output_schemas_module", @@ -131,7 +131,7 @@ "sha256": "ef40501b2afbe4cd2adfa80480344783bbec1a00e8e9850086a5d044231d1f53" } ], - "validator_source_set_sha256": "afb4e11f1360d5732f977351c7787117e18c70aaf90f80ad02bb45572ee3ce59", + "validator_source_set_sha256": "6e875209853d97288e5eae1464c0c3a333f37b31ad3467839954aa4edbf657c4", "pdfjs_version": "5.4.624", "generation_contract": { "source_path": "source.pdf",