diff --git a/docs/HEADERLESS_TABLES.md b/docs/HEADERLESS_TABLES.md index c2687fe..776567d 100644 --- a/docs/HEADERLESS_TABLES.md +++ b/docs/HEADERLESS_TABLES.md @@ -2,7 +2,7 @@ `convert_pdf_to_markdown` accepts the explicit, default-off `allow_headerless_tables: true` option. The selected renderer profile is -`1.22.0`; ordinary calls and explicit `false` retain profile `1.21.0` and +`1.23.0`; ordinary calls and explicit `false` retain profile `1.21.0` and the previous complete result envelope. This addresses a narrow representational limit: GFM assigns header semantics @@ -21,6 +21,15 @@ This does not broaden the captioned/solid-mask, segmented-rule, or unruled table paths. Those and existing GFM tables keep their prior rules. Source text is HTML-escaped, never interpreted as markup or an attribute. +Profile `1.23.0` uses each complete text advance box when choosing its row, +so tightly spaced rows do not become ambiguous merely because the top of +one box is within the preceding row's tolerance. A single explicit `Table` +caption may precede the complete grid in the same text run when its whole +line is outside the grid. The caption stays ordinary text and is excluded +from the cell proposal. Other extra rows, overlapping captions and text +crossing cells still prevent conversion. The earlier `1.22.0` profile could +reject these complete captioned grids; no header role is added by this repair. + Example tool arguments: ```json diff --git a/docs/MCP_CONTRACT.md b/docs/MCP_CONTRACT.md index cdc6fa5..490b018 100644 --- a/docs/MCP_CONTRACT.md +++ b/docs/MCP_CONTRACT.md @@ -93,7 +93,7 @@ entrypoint with bounded value, history, citation, and page-coverage views. `convert_pdf_to_markdown` has a default-off `allow_headerless_tables` option for narrowly qualified complete ruled-rectangle grids. The opted-in renderer -profile `1.22.0` emits escaped HTML data cells without assigning header roles; +profile `1.23.0` emits escaped HTML data cells without assigning header roles; typed uncertainty remains. Omitted or false retains the unchanged default profile `1.21.0`. See [Header-neutral tables](HEADERLESS_TABLES.md). diff --git a/pdf-toolkit-mcp-share/server/index.js b/pdf-toolkit-mcp-share/server/index.js index 2f49d54..11d0721 100755 --- a/pdf-toolkit-mcp-share/server/index.js +++ b/pdf-toolkit-mcp-share/server/index.js @@ -3788,7 +3788,7 @@ async function listTools(request) { compact: { type: "boolean", description: "Opt into counted dot-leader, isolated page-number, and Unicode-letter spaced-hyphen normalizations. Default: false." }, remove_page_furniture: { type: "boolean", description: "Remove only source-evidenced extreme-margin page furniture and report every removed line by typed gap and counts. Set false to preserve all source lines. Default: true." }, emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, - allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.22.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." }, + allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.23.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." }, output_path: { type: "string", description: "Optional absolute .md path, or ~/ path. The file is written only after complete bytes are staged and verified." }, overwrite: { type: "boolean", description: "Replace an existing output_path only when its exact expected_output_identity is also supplied. Default: false." }, expected_output_identity: EXPECTED_MARKDOWN_OUTPUT_IDENTITY_INPUT_SCHEMA diff --git a/pdf-toolkit-mcp-share/server/markdown-conversion.js b/pdf-toolkit-mcp-share/server/markdown-conversion.js index bf6c9ee..c5a0443 100644 --- a/pdf-toolkit-mcp-share/server/markdown-conversion.js +++ b/pdf-toolkit-mcp-share/server/markdown-conversion.js @@ -8,7 +8,7 @@ const RENDERER = Object.freeze({ name: "pdf-tools.layout-markdown-renderer", version: "1.21.0", }); -const HEADERLESS_RENDERER = Object.freeze({ ...RENDERER, version: "1.22.0" }); +const HEADERLESS_RENDERER = Object.freeze({ ...RENDERER, version: "1.23.0" }); const SUPPORTED_LAYOUT_IR_VERSION = "1.6.0"; // Bounded geometric table inference. A run of adjacent lines is treated as a @@ -2111,6 +2111,10 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables = } if (filledCells / totalCells < RECT_MIN_FILL_RATIO) return { reason: "ruling_unsupported" }; + const candidateRows = lineRecords + .toSorted((left, right) => left.line.y - right.line.y) + .map(record => ({ line: record.line, cells: record.structuralInsideNonWhitespace })); + const headerless = !hasHeaderEvidence(candidateRows) && !hasFirstRowBandEvidence(bands); const cells = Array.from({ length: rowCount }, () => ( Array.from({ length: columnCount }, () => []) )); @@ -2129,7 +2133,8 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables = if (centerX >= xEdges[column] - RECT_CELL_ASSIGNMENT_SLACK && centerX <= xEdges[column + 1] + RECT_CELL_ASSIGNMENT_SLACK && box.y >= yEdges[row] - RECT_CELL_ASSIGNMENT_SLACK - && box.y <= yEdges[row + 1] + RECT_CELL_ASSIGNMENT_SLACK) { + && (allowHeaderlessTables && headerless ? rectBottom(box) : box.y) + <= yEdges[row + 1] + RECT_CELL_ASSIGNMENT_SLACK) { matches.push({ row, column }); } } @@ -2184,10 +2189,6 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables = if (emptyColumns.slice(firstColumn, lastColumn + 1).some(Boolean)) return { reason: "ruling_unsupported" }; if (firstColumn > lastColumn) return { reason: "ruling_unsupported" }; const trimmedGrid = grid.map(row => row.slice(firstColumn, lastColumn + 1)); - const candidateRows = lineRecords - .sort((left, right) => left.line.y - right.line.y) - .map(record => ({ line: record.line, cells: record.structuralInsideNonWhitespace })); - const headerless = !hasHeaderEvidence(candidateRows) && !hasFirstRowBandEvidence(bands); if (headerless && !allowHeaderlessTables) { return { reason: "header" }; } @@ -2197,8 +2198,18 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables = // A partial rectangle census or a center-only cell assignment cannot // establish the complete grid this profile promises to preserve. const candidateLineIds = new Set(candidateRows.map(row => row.line.id)); - if (candidateLineIds.size !== runLineIds.size - || [...runLineIds].some(id => !candidateLineIds.has(id))) { + const first = run.findIndex(row => candidateLineIds.has(row.line.id)); + const last = run.findLastIndex(row => candidateLineIds.has(row.line.id)); + const gridBox = { x: bounds.x, y: bounds.y, + width: bounds.right - bounds.x, height: bounds.bottom - bounds.y }; + // One explicit caption can share the run with the grid. Keep it as + // text only when it precedes the complete contiguous grid and lies + // wholly outside its envelope. Other extra rows remain ambiguous. + if (first < 0 || last - first + 1 !== candidateLineIds.size + || run.slice(first, last + 1).some(row => !candidateLineIds.has(row.line.id)) + || last !== run.length - 1 || first > 1 + || (first === 1 && (!/^table\s+(?:\d+|[ivxlcdm]+)(?:[:.]|\s|$)/iu.test(run[0].line.text.trim()) + || boxesOverlap(run[0].line, gridBox)))) { return { reason: "topology" }; } // Vector paint makes an otherwise fully observed text page "partial" @@ -2225,7 +2236,8 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables = } } // Keep empty outer slots rather than trimming source grid columns. - return { kind: "table", grid, rows: candidateRows, headerless: true }; + return { kind: "table", grid, rows: candidateRows, headerless: true, + leadingRows: run.slice(0, first) }; } return { kind: "table", grid: trimmedGrid, rows: candidateRows }; } @@ -2670,10 +2682,12 @@ function segmentTextRows(page, rows, itemById, ruledClusters, allowHeaderlessTab ? tryBuildRectGrid(page, run, ruledClusters, itemById, allowHeaderlessTables) : null; if (ruling?.kind === "table") { - segments.push({ kind: "table", rows: run, grid: ruling.grid, ...(ruling.headerless ? { headerless: true } : {}) }); + if (ruling.leadingRows?.length) segments.push({ kind: "text", rows: ruling.leadingRows }); + segments.push({ kind: "table", rows: ruling.headerless ? ruling.rows : run, + grid: ruling.grid, ...(ruling.headerless ? { headerless: true } : {}) }); if (ruling.headerless) { tableReason = "header"; - regions.push({ run, reason: "header" }); + regions.push({ run: ruling.rows, reason: "header" }); } } else { // Preserve the reason from the rect-evidence attempt. The text path diff --git a/pdf-toolkit-mcp-share/server/output-schemas.js b/pdf-toolkit-mcp-share/server/output-schemas.js index 57b04ff..7bcc087 100644 --- a/pdf-toolkit-mcp-share/server/output-schemas.js +++ b/pdf-toolkit-mcp-share/server/output-schemas.js @@ -1503,7 +1503,7 @@ export const TOOL_SUCCESS_OUTPUT_SCHEMAS = Object.freeze({ { renderer: object({ name: { const: "pdf-tools.layout-markdown-renderer" }, - version: enumString(["1.21.0", "1.22.0"]), + version: enumString(["1.21.0", "1.23.0"]), }), conversion_status: enumString(["complete", "partial", "failed"]), markdown: string, diff --git a/scripts/macos-claude-installed-shannon.mjs b/scripts/macos-claude-installed-shannon.mjs index b53f10c..95ed8d5 100755 --- a/scripts/macos-claude-installed-shannon.mjs +++ b/scripts/macos-claude-installed-shannon.mjs @@ -23,7 +23,7 @@ if (!process.argv[2] || !process.argv[3]) { // const EXPECTED_SOURCE_SHA256 = "6e4e3411984f3edf99dbfe8b941cb5e8a321379ff0cae6ae5c1f592ad8882ca8"; const EXPECTED_MARKDOWN_SHA256 = "b9bd7ca7db26fe807f294565c39de8d4fe9337908afa74cd7e2896a800c92991"; -const EXPECTED_TOOL_CONTRACT_SHA256 = "fccd8d1a5387816d4407868b8bebe26865b7e81318b5f20fb6ebb6209fc56386"; +const EXPECTED_TOOL_CONTRACT_SHA256 = "d5e40605f3f8d0fa656a798ed38400b3f12981250d8fcc9b05ff094a861dc17f"; const EXPECTED_PAGE_COUNT = 55; const EXPECTED_GAP_COUNT = 26; const EXPECTED_REPLACEMENT_CHARACTER_COUNT = 6; diff --git a/scripts/macos-claude-installed-smoke.mjs b/scripts/macos-claude-installed-smoke.mjs index 00872b3..4e8c3f4 100755 --- a/scripts/macos-claude-installed-smoke.mjs +++ b/scripts/macos-claude-installed-smoke.mjs @@ -33,7 +33,7 @@ const textFixture = path.join(fixtureDirectory, "synthetic-text-two-page.pdf"); const rasterFixture = path.join(fixtureDirectory, "synthetic-raster-only.pdf"); const mutationDirectory = path.join(fixtureDirectory, "mutation-output"); const toolNames = []; -const EXPECTED_TOOL_CONTRACT_SHA256 = "fccd8d1a5387816d4407868b8bebe26865b7e81318b5f20fb6ebb6209fc56386"; +const EXPECTED_TOOL_CONTRACT_SHA256 = "d5e40605f3f8d0fa656a798ed38400b3f12981250d8fcc9b05ff094a861dc17f"; const ACCESSIBILITY_CONCLUSION_KEYS = Object.freeze([ "certification", "document_accessibility", diff --git a/server/index.js b/server/index.js index 2f49d54..11d0721 100755 --- a/server/index.js +++ b/server/index.js @@ -3788,7 +3788,7 @@ async function listTools(request) { compact: { type: "boolean", description: "Opt into counted dot-leader, isolated page-number, and Unicode-letter spaced-hyphen normalizations. Default: false." }, remove_page_furniture: { type: "boolean", description: "Remove only source-evidenced extreme-margin page furniture and report every removed line by typed gap and counts. Set false to preserve all source lines. Default: true." }, emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, - allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.22.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." }, + allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.23.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." }, output_path: { type: "string", description: "Optional absolute .md path, or ~/ path. The file is written only after complete bytes are staged and verified." }, overwrite: { type: "boolean", description: "Replace an existing output_path only when its exact expected_output_identity is also supplied. Default: false." }, expected_output_identity: EXPECTED_MARKDOWN_OUTPUT_IDENTITY_INPUT_SCHEMA diff --git a/server/markdown-conversion.js b/server/markdown-conversion.js index bf6c9ee..c5a0443 100644 --- a/server/markdown-conversion.js +++ b/server/markdown-conversion.js @@ -8,7 +8,7 @@ const RENDERER = Object.freeze({ name: "pdf-tools.layout-markdown-renderer", version: "1.21.0", }); -const HEADERLESS_RENDERER = Object.freeze({ ...RENDERER, version: "1.22.0" }); +const HEADERLESS_RENDERER = Object.freeze({ ...RENDERER, version: "1.23.0" }); const SUPPORTED_LAYOUT_IR_VERSION = "1.6.0"; // Bounded geometric table inference. A run of adjacent lines is treated as a @@ -2111,6 +2111,10 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables = } if (filledCells / totalCells < RECT_MIN_FILL_RATIO) return { reason: "ruling_unsupported" }; + const candidateRows = lineRecords + .toSorted((left, right) => left.line.y - right.line.y) + .map(record => ({ line: record.line, cells: record.structuralInsideNonWhitespace })); + const headerless = !hasHeaderEvidence(candidateRows) && !hasFirstRowBandEvidence(bands); const cells = Array.from({ length: rowCount }, () => ( Array.from({ length: columnCount }, () => []) )); @@ -2129,7 +2133,8 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables = if (centerX >= xEdges[column] - RECT_CELL_ASSIGNMENT_SLACK && centerX <= xEdges[column + 1] + RECT_CELL_ASSIGNMENT_SLACK && box.y >= yEdges[row] - RECT_CELL_ASSIGNMENT_SLACK - && box.y <= yEdges[row + 1] + RECT_CELL_ASSIGNMENT_SLACK) { + && (allowHeaderlessTables && headerless ? rectBottom(box) : box.y) + <= yEdges[row + 1] + RECT_CELL_ASSIGNMENT_SLACK) { matches.push({ row, column }); } } @@ -2184,10 +2189,6 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables = if (emptyColumns.slice(firstColumn, lastColumn + 1).some(Boolean)) return { reason: "ruling_unsupported" }; if (firstColumn > lastColumn) return { reason: "ruling_unsupported" }; const trimmedGrid = grid.map(row => row.slice(firstColumn, lastColumn + 1)); - const candidateRows = lineRecords - .sort((left, right) => left.line.y - right.line.y) - .map(record => ({ line: record.line, cells: record.structuralInsideNonWhitespace })); - const headerless = !hasHeaderEvidence(candidateRows) && !hasFirstRowBandEvidence(bands); if (headerless && !allowHeaderlessTables) { return { reason: "header" }; } @@ -2197,8 +2198,18 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables = // A partial rectangle census or a center-only cell assignment cannot // establish the complete grid this profile promises to preserve. const candidateLineIds = new Set(candidateRows.map(row => row.line.id)); - if (candidateLineIds.size !== runLineIds.size - || [...runLineIds].some(id => !candidateLineIds.has(id))) { + const first = run.findIndex(row => candidateLineIds.has(row.line.id)); + const last = run.findLastIndex(row => candidateLineIds.has(row.line.id)); + const gridBox = { x: bounds.x, y: bounds.y, + width: bounds.right - bounds.x, height: bounds.bottom - bounds.y }; + // One explicit caption can share the run with the grid. Keep it as + // text only when it precedes the complete contiguous grid and lies + // wholly outside its envelope. Other extra rows remain ambiguous. + if (first < 0 || last - first + 1 !== candidateLineIds.size + || run.slice(first, last + 1).some(row => !candidateLineIds.has(row.line.id)) + || last !== run.length - 1 || first > 1 + || (first === 1 && (!/^table\s+(?:\d+|[ivxlcdm]+)(?:[:.]|\s|$)/iu.test(run[0].line.text.trim()) + || boxesOverlap(run[0].line, gridBox)))) { return { reason: "topology" }; } // Vector paint makes an otherwise fully observed text page "partial" @@ -2225,7 +2236,8 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables = } } // Keep empty outer slots rather than trimming source grid columns. - return { kind: "table", grid, rows: candidateRows, headerless: true }; + return { kind: "table", grid, rows: candidateRows, headerless: true, + leadingRows: run.slice(0, first) }; } return { kind: "table", grid: trimmedGrid, rows: candidateRows }; } @@ -2670,10 +2682,12 @@ function segmentTextRows(page, rows, itemById, ruledClusters, allowHeaderlessTab ? tryBuildRectGrid(page, run, ruledClusters, itemById, allowHeaderlessTables) : null; if (ruling?.kind === "table") { - segments.push({ kind: "table", rows: run, grid: ruling.grid, ...(ruling.headerless ? { headerless: true } : {}) }); + if (ruling.leadingRows?.length) segments.push({ kind: "text", rows: ruling.leadingRows }); + segments.push({ kind: "table", rows: ruling.headerless ? ruling.rows : run, + grid: ruling.grid, ...(ruling.headerless ? { headerless: true } : {}) }); if (ruling.headerless) { tableReason = "header"; - regions.push({ run, reason: "header" }); + regions.push({ run: ruling.rows, reason: "header" }); } } else { // Preserve the reason from the rect-evidence attempt. The text path diff --git a/server/output-schemas.js b/server/output-schemas.js index 57b04ff..7bcc087 100644 --- a/server/output-schemas.js +++ b/server/output-schemas.js @@ -1503,7 +1503,7 @@ export const TOOL_SUCCESS_OUTPUT_SCHEMAS = Object.freeze({ { renderer: object({ name: { const: "pdf-tools.layout-markdown-renderer" }, - version: enumString(["1.21.0", "1.22.0"]), + version: enumString(["1.21.0", "1.23.0"]), }), conversion_status: enumString(["complete", "partial", "failed"]), markdown: string, diff --git a/test/convert-pdf-to-markdown.test.js b/test/convert-pdf-to-markdown.test.js index 5c1b800..413ffc2 100644 --- a/test/convert-pdf-to-markdown.test.js +++ b/test/convert-pdf-to-markdown.test.js @@ -67,16 +67,22 @@ const makeUniformGridFixture = target => drawGrid(target, { headerSize: 11, sepa // Identical size, but the header uses a second embed of the same visible font. const makeDoubleEmbedGridFixture = target => drawGrid(target, { headerSize: 11, separateHeaderResource: true }); -async function makeHeaderlessRectGridFixture(targetPath) { +async function makeHeaderlessRectGridFixture(targetPath, { tight = false } = {}) { const document = await PDFDocument.create(); const page = document.addPage([612, 792]); const font = await document.embedFont(StandardFonts.Helvetica); const rows = [["Region", "Q1", "Q2"], ["&|'", "1200", "1450"], ["South", "980", "1020"], ["West", "1500", "1380"]]; - rows.forEach((row, r) => row.forEach((value, c) => { - page.drawRectangle({ x: 72 + c * 150, y: 680 - (r + 1) * 48, - width: 150, height: 48, borderWidth: 1, borderColor: rgb(0, 0, 0) }); - page.drawText(value, { x: 82 + c * 150, y: 650 - r * 48, size: 12, font }); + const rowHeight = tight ? 10 : 48; + if (tight) { + page.drawText("Table 1:", { x: 82, y: 700, size: 8, font }); + page.drawText("Quarterly values", { x: 260, y: 700, size: 8, font }); + } + rows.forEach((row, r) => (tight ? [2, 1, 0] : [0, 1, 2]).forEach(c => { + page.drawRectangle({ x: 72 + c * 150, y: 680 - (r + 1) * rowHeight, + width: 150, height: rowHeight, borderWidth: 1, borderColor: rgb(0, 0, 0) }); + page.drawText(row[c], { x: 82 + c * 150, y: (tight ? 671.6 : 650) - r * rowHeight, + size: tight ? 8 : 12, font }); })); await fs.writeFile(targetPath, await document.save({ useObjectStreams: false })); } @@ -250,6 +256,7 @@ describe("convert_pdf_to_markdown MCP tool", () => { let rotatedLinkFixture; let hostileLinkFixture; let headerlessRectFixture; + let tightHeaderlessRectFixture; beforeAll(async () => { temporaryRoot = await fs.mkdtemp(path.join(os.tmpdir(), "pdf-tools-markdown-")); @@ -265,6 +272,7 @@ describe("convert_pdf_to_markdown MCP tool", () => { rotatedLinkFixture = path.join(temporaryRoot, "link-rotated.pdf"); hostileLinkFixture = path.join(temporaryRoot, "link-hostile.pdf"); headerlessRectFixture = path.join(temporaryRoot, "headerless-rect.pdf"); + tightHeaderlessRectFixture = path.join(temporaryRoot, "tight-headerless-rect.pdf"); await makeStructureFixture(structureFixture); await makeDenseFixture(denseFixture); await makeGeometryFixture(geometryFixture); @@ -277,6 +285,7 @@ describe("convert_pdf_to_markdown MCP tool", () => { await makeLinkFixture(rotatedLinkFixture, { rotation: 90, crop: true }); await makeLinkFixture(hostileLinkFixture, { url: "https://example.com/a(b)c" }); await makeHeaderlessRectGridFixture(headerlessRectFixture); + await makeHeaderlessRectGridFixture(tightHeaderlessRectFixture, { tight: true }); transport = new StdioClientTransport({ command: process.execPath, args: [path.join(REPO_ROOT, "server/index.js")], @@ -434,7 +443,7 @@ describe("convert_pdf_to_markdown MCP tool", () => { const opted = await client.callTool({ name: "convert_pdf_to_markdown", arguments: { ...args, allow_headerless_tables: true, emit_table_proposals: true, compact: true } }); expect(opted.isError).not.toBe(true); - expect(opted.structuredContent.renderer.version).toBe("1.22.0"); + expect(opted.structuredContent.renderer.version).toBe("1.23.0"); expect(opted.structuredContent.options.allow_headerless_tables).toBe(true); expect(opted.structuredContent.markdown).toContain("<img>&|'12001450"); expect(opted.structuredContent.markdown).not.toContain(""); @@ -446,6 +455,28 @@ describe("convert_pdf_to_markdown MCP tool", () => { expect(rejected.isError).toBe(true); }, 30_000); + it("retains a real tight captioned grid through the MCP tool", async () => { + const before = await fs.readFile(tightHeaderlessRectFixture); + const args = { pdf_path: tightHeaderlessRectFixture, max_markdown_bytes: 100000 }; + const baseline = await client.callTool({ name: "convert_pdf_to_markdown", arguments: args }); + const result = await client.callTool({ name: "convert_pdf_to_markdown", arguments: { + ...args, allow_headerless_tables: true, emit_table_proposals: true, + } }); + expect(result.isError).not.toBe(true); + expect(baseline.structuredContent.markdown).not.toContain(""); + const converted = result.structuredContent; + expect(converted.renderer.version).toBe("1.23.0"); + expect(converted.markdown.match(/
/gu)).toHaveLength(1); + expect(converted.markdown).toContain(""); + expect(converted.markdown).toContain(""); + expect(converted.markdown.split("Table 1:")).toHaveLength(2); + expect(converted.markdown.indexOf("Table 1:")).toBeLessThan(converted.markdown.indexOf("
RegionQ1Q2
West15001380
")); + expect(converted.markdown).not.toContain(""); expect(result.markdown).not.toContain("
"); + expect(converted.conversion_status).toBe("partial"); + expect(converted.table_proposals).toHaveLength(1); + expect(await fs.readFile(tightHeaderlessRectFixture)).toEqual(before); + }, 30_000); + it("escapes a table cell delimiter exactly once", async () => { const result = await client.callTool({ name: "convert_pdf_to_markdown", diff --git a/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json b/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json index 01394fc..de88001 100644 --- a/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json +++ b/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json @@ -79,14 +79,14 @@ { "role": "markdown_conversion_module", "path": "server/markdown-conversion.js", - "bytes": 186425, - "sha256": "cd8da7d747101cf746a2ac72336627758cba4feaab47b7fc9672cf2631e31b9c" + "bytes": 187442, + "sha256": "59fe4915bc32fe734755d12ebec0df2d9829a907182e2a796fbb4c7309b5a538" }, { "role": "output_schemas_module", "path": "server/output-schemas.js", "bytes": 84378, - "sha256": "4a522decf2d5338ca48f8383c188a36719c9ee88a702791c3f6c1c8f29844f77" + "sha256": "8a906a082d2b4d5541b9d2a98f07faba1ca40380c236879b5939da4102fa3941" }, { "role": "package_json", @@ -131,7 +131,7 @@ "sha256": "ef40501b2afbe4cd2adfa80480344783bbec1a00e8e9850086a5d044231d1f53" } ], - "validator_source_set_sha256": "a67e5beb7c30a8b52c90fe7dc388e628b004b5833539cf1faa4aa8e7b450baad", + "validator_source_set_sha256": "3df238fa35763b4461e65a1a884572d586646401415c32b9be98e413dc389a26", "pdfjs_version": "5.4.624", "generation_contract": { "source_path": "source.pdf", diff --git a/test/fixtures/eval/trajectories/tool-contracts.v3.json b/test/fixtures/eval/trajectories/tool-contracts.v3.json index 672ea49..9a8ed41 100644 --- a/test/fixtures/eval/trajectories/tool-contracts.v3.json +++ b/test/fixtures/eval/trajectories/tool-contracts.v3.json @@ -5,7 +5,7 @@ "name": "pdf-tools", "version": "0.14.0" }, - "tools_sha256": "179dc9340e4554d8c6bbae009ce3fb202e8693ac96fefedc065adb1ab0ea233d", + "tools_sha256": "13dd65eed63634710f7e87327950d3043ea5d30008809e89ad0ee83065c11574", "tools": [ { "name": "add_signature_field", @@ -592,7 +592,7 @@ }, "allow_headerless_tables": { "type": "boolean", - "description": "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.22.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." + "description": "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.23.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." }, "output_path": { "type": "string", diff --git a/test/markdown-conversion.test.js b/test/markdown-conversion.test.js index e1c3a53..c9d94c5 100644 --- a/test/markdown-conversion.test.js +++ b/test/markdown-conversion.test.js @@ -428,6 +428,16 @@ describe("layout Markdown renderer", () => { expect(first.conversion_status).toBe("complete"); }); + it("keeps established header-evidenced GFM cells unchanged in the opt-in profile", async () => { + const layout = attachRuledRects(await validatedSyntheticLayout([{ items: ruledGridItems() }]), ruledGridRects()); + const baseline = renderPdfLayoutToMarkdown(layout); + const opted = renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: true }); + expect(opted.pages).toEqual(baseline.pages); + expect(opted.gaps).toEqual(baseline.gaps); + expect(opted.markdown.split("## Conversion limitations")[0]) + .toEqual(baseline.markdown.split("## Conversion limitations")[0]); + }); + it("keeps the complete default envelope unchanged when header-neutral mode is off", async () => { const layout = attachRuledRects(await validatedSyntheticLayout([{ items: ruledGridItems({ headerSize: 12 }), @@ -447,7 +457,7 @@ describe("layout Markdown renderer", () => { const result = renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: true, emitTableProposals: true, compact: true, }); - expect(result.renderer.version).toBe("1.22.0"); + expect(result.renderer.version).toBe("1.23.0"); expect(result.options.allow_headerless_tables).toBe(true); expect(result.markdown).toContain("
North12001450
"); @@ -458,6 +468,61 @@ describe("layout Markdown renderer", () => { expect(() => validateMarkdownConversionSemantics(result, { layout })).not.toThrow(); }); + it.each([false, true])("retains tightly spaced complete cells with caption=%s", async caption => { + const values = [["Label", "First", "Second"], ["North", "1200", "1450"], + ["South", "980", "1020"], ["West", "1500", "1380"]]; + // The PDF paints each row right to left. Each body text box starts within + // the preceding cell's old top-only tolerance but fits one whole cell. + const items = values.flatMap((row, rowIndex) => row.toReversed().map((value, index) => textItem(value, { + top: 110.7 + rowIndex * 10, left: 82 + (2 - index) * 150, + fontSize: 8, eol: index === 2, + }))); + if (caption) items.unshift(textItem("Table 1:", { top: 90, left: 82, fontSize: 8, eol: false }), + textItem("Quarterly values", { top: 90, left: 260, fontSize: 8 })); + const rects = values.flatMap((row, rowIndex) => row.map((_, column) => ({ + x: 72 + column * 150, y: 112 + rowIndex * 10, width: 150, height: 10, verb: "stroke", + }))); + const layout = attachRuledRects(await validatedSyntheticLayout([{ items }]), rects); + const baseline = renderPdfLayoutToMarkdown(layout); + const result = renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: true, emitTableProposals: true }); + expect(baseline.markdown).not.toContain(""); + expect(result.markdown.match(/
/gu)).toHaveLength(1); + expect(result.markdown).toContain(""); + expect(result.markdown).toContain(""); + expect(result.markdown).toContain(""); + if (caption) { + expect(result.markdown.split("Table 1:")).toHaveLength(2); + expect(result.markdown.indexOf("Table 1:")).toBeLessThan(result.markdown.indexOf("
LabelFirstSecond
North12001450
West15001380
")); + expect(result.table_proposal_regions[0].text_items.some(item => item.text.includes("Table 1:"))).toBe(false); + } + expect(result.conversion_status).toBe("partial"); + expect(result.markdown).not.toContain("
"); + expect(renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: false })).toEqual(baseline); + expect(() => validateMarkdownConversionSemantics(result, { layout })).not.toThrow(); + }); + + it.each(["overlapping_caption", "ordinary_prose", "cross_row_text"])( + "does not force a tight grid through %s", async attack => { + const items = ruledGridItems({ headerSize: 12 }); + if (attack === "cross_row_text") { + const north = items.find(item => item.str === "North"); + north.height = 65; + north.transform[3] = 65; + } else { + items.unshift(textItem(attack === "ordinary_prose" ? "Section:" : "Table 1:", { + top: attack === "overlapping_caption" ? 106 : 90, left: 82, fontSize: 8, eol: false, + }), textItem("Quarterly values", { + top: attack === "overlapping_caption" ? 106 : 90, left: 260, fontSize: 8, + })); + } + const layout = attachRuledRects(await validatedSyntheticLayout([{ items }]), headerlessGridRects()); + const result = renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: true }); + expect(result.markdown).not.toContain(""); + expect(result.markdown).toContain("North"); + expect(result.markdown).toContain("West"); + }, + ); + it("escapes source markup without interpreting pipe characters or creating HTML attributes", async () => { const items = ruledGridItems({ headerSize: 12 }).map(item => item.str === "North" ? textItem("&|\"'", { top: 178, left: 82, fontSize: 12, eol: false }) : item); diff --git a/test/mcp-contract.test.js b/test/mcp-contract.test.js index 028d2d1..1b4282f 100644 --- a/test/mcp-contract.test.js +++ b/test/mcp-contract.test.js @@ -240,7 +240,8 @@ const EXAMPLE_PDF = path.join(REPO_ROOT, "example-fw9.pdf"); // renderer-profile/output-schema contract. Default conversion bytes unchanged. // 2026-10-20: prepare_lumin_request is stateful/non-idempotent; fetch is // neutrally described and no longer exposes arbitrary credential headers. -const TOOL_CONTRACT_SHA256 = "fccd8d1a5387816d4407868b8bebe26865b7e81318b5f20fb6ebb6209fc56386"; +// 2026-10-20: version the optional tight-grid/caption repair as renderer 1.23.0. +const TOOL_CONTRACT_SHA256 = "d5e40605f3f8d0fa656a798ed38400b3f12981250d8fcc9b05ff094a861dc17f"; const CLOSED_READ = Object.freeze({ readOnlyHint: true,