From 21a467128da7812ef1ccdcd2086352e19ba02667 Mon Sep 17 00:00:00 2001 From: David Fant Date: Mon, 26 Oct 2026 18:56:59 -0700 Subject: [PATCH 1/4] Support parser-owned literal ranges during Markdown repair --- .changeset/protected-literal-repair.md | 8 ++ packages/remend/README.md | 21 +++++ .../remend/__tests__/literal-ranges.test.ts | 78 +++++++++++++++++++ packages/remend/src/emphasis-handlers.ts | 8 +- packages/remend/src/index.ts | 32 ++++++-- packages/remend/src/link-image-handler.ts | 20 +++-- packages/remend/src/patterns.ts | 60 +++++++++++++- packages/remend/src/scan.ts | 36 ++++++++- packages/remend/src/strikethrough-handler.ts | 4 +- packages/remend/src/utils.ts | 11 ++- 10 files changed, 256 insertions(+), 22 deletions(-) create mode 100644 .changeset/protected-literal-repair.md create mode 100644 packages/remend/__tests__/literal-ranges.test.ts diff --git a/.changeset/protected-literal-repair.md b/.changeset/protected-literal-repair.md new file mode 100644 index 00000000..8220459a --- /dev/null +++ b/.changeset/protected-literal-repair.md @@ -0,0 +1,8 @@ +--- +"remend": patch +--- + +Allow syntax extensions to supply opaque source ranges through `literalRanges`. +Incomplete-Markdown repair preserves those ranges while retaining full-document +formatting context. Range callbacks use the current source after each edit, and +nested calls and exceptions restore the caller's scanner scope. diff --git a/packages/remend/README.md b/packages/remend/README.md index 5b9b44a2..f3481248 100644 --- a/packages/remend/README.md +++ b/packages/remend/README.md @@ -215,3 +215,24 @@ Remend is built for high-performance streaming scenarios: - Early returns for common cases For more info, see the [documentation](https://streamdown.ai/docs/termination). + + +### Parser-owned literal ranges + +`literalRanges` optionally supplies opaque source intervals to the existing repair +scanner. Built-in handlers keep the full Markdown context but do not interpret or +edit those literal bodies. Streamdown supplies ranges from its registered-container +micromark grammar, not a second tag matcher. + +```ts +remend(source, { + literalRanges: currentSource => parserLiteralRanges(currentSource), +}); +``` + +The callback returns UTF-16 `{ start, end }` offsets, with an exclusive end, for +sorted, non-overlapping intervals. It is cached with the current source scan and +called again when an earlier handler changes the source. Scope is synchronous and +restored after nested calls or exceptions. There is no source encoding or +fragment-by-fragment repair. User-defined custom handlers still receive the entire +string and remain responsible for respecting literal source themselves. diff --git a/packages/remend/__tests__/literal-ranges.test.ts b/packages/remend/__tests__/literal-ranges.test.ts new file mode 100644 index 00000000..16d904f4 --- /dev/null +++ b/packages/remend/__tests__/literal-ranges.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import remend from "../src/index"; + +function literalRanges(text: string) { + const start = text.indexOf(""); + const close = text.indexOf("", start); + return start < 0 + ? [] + : [{ start, end: close < 0 ? text.length : close + "".length }]; +} +describe("parser-owned opaque ranges", () => { + it.each([ + "20~25", + "[unclosed](https://example.com", + "**raw", + "`raw", + "$$math", + "\n- > 25", + "\n```\ncode", + "trailing ", + ])("never heals literal source %j", (body) => { + const source = `${body}`; + expect(remend(source, { literalRanges })).toBe(source); + }); + it("preserves full surrounding Markdown context", () => { + const source = "**Before **raw after**"; + expect(remend(source, { literalRanges })).toBe(source); + expect( + remend("**Before **raw after", { literalRanges }) + ).toBe("**Before **raw after**"); + }); + it("ignores literal brackets while healing a crossing link", () => { + const source = "[Before ][fake](raw after"; + expect(remend(source, { literalRanges })).toBe( + source + "](streamdown:incomplete-link)" + ); + }); + it("recomputes offsets when earlier repairs insert characters", () => { + expect( + remend("20~25 20~25 **raw", { literalRanges }) + ).toBe("20\\~25 20~25 **raw"); + }); + it("does not trim whitespace owned by an unfinished literal", () => { + expect(remend("raw ", { literalRanges })).toBe("raw "); + }); + it("restores scoped ranges after nested calls", () => { + const result = remend("20~25", { + literalRanges, + handlers: [ + { + name: "nested", + handle: (text) => { + expect(remend("20~25")).toBe("20\\~25"); + return text; + }, + }, + ], + }); + expect(result).toBe("20~25"); + expect(remend("20~25")).toBe("20\\~25"); + }); + it("clears scoped ranges after a handler throws", () => { + expect(() => + remend("raw", { + literalRanges, + handlers: [ + { + name: "throw", + handle: () => { + throw new Error("test"); + }, + }, + ], + }) + ).toThrow("test"); + expect(remend("20~25")).toBe("20\\~25"); + }); +}); diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 8905ae68..33229608 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -398,7 +398,7 @@ export const handleIncompleteBold = (text: string): string => { } const contentAfterMarker = boldMatch[2]; - const markerIndex = text.lastIndexOf(boldMatch[1]); + const markerIndex = boldMatch.index!; // Check if the bold marker is within a code block (fenced or inline) if ( @@ -469,7 +469,7 @@ export const handleIncompleteDoubleUnderscoreItalic = ( halfCompleteUnderscorePattern ); if (halfCompleteMatch) { - const markerIndex = text.lastIndexOf(halfCompleteMatch[1]); + const markerIndex = halfCompleteMatch.index!; if ( !( isInsideCodeBlock(text, markerIndex) || @@ -484,7 +484,7 @@ export const handleIncompleteDoubleUnderscoreItalic = ( } const contentAfterMarker = italicMatch[2]; - const markerIndex = text.lastIndexOf(italicMatch[1]); + const markerIndex = italicMatch.index!; // Check if the italic marker is within a code block (fenced or inline) if ( @@ -770,7 +770,7 @@ export const handleIncompleteBoldItalic = (text: string): string => { } const contentAfterMarker = boldItalicMatch[2]; - const markerIndex = text.lastIndexOf(boldItalicMatch[1]); + const markerIndex = boldItalicMatch.index!; if (shouldSkipBoldItalicCompletion(text, contentAfterMarker, markerIndex)) { return text; diff --git a/packages/remend/src/index.ts b/packages/remend/src/index.ts index 0ecf9137..d181f4cd 100644 --- a/packages/remend/src/index.ts +++ b/packages/remend/src/index.ts @@ -17,7 +17,13 @@ import { INCOMPLETE_IMAGE_PLACEHOLDER, type LinkMode, } from "./link-image-handler"; -import { clearScanCache } from "./scan"; +import { + clearScanCache, + getScan, + REGION, + withLiteralRanges, + type LiteralRanges, +} from "./scan"; import { handleIncompleteSetextHeading } from "./setext-heading-handler"; import { handleSingleTildeEscape } from "./single-tilde-handler"; import { handleIncompleteStrikethrough } from "./strikethrough-handler"; @@ -52,6 +58,8 @@ export interface RemendHandler { * Set an option to `false` to disable that specific completion. */ export interface RemendOptions { + /** Parser-owned opaque source ranges, recalculated when a handler edits text. */ + literalRanges?: LiteralRanges; /** Complete bold formatting (e.g., `**text` → `**text**`) */ bold?: boolean; /** Complete bold-italic formatting (e.g., `***text` → `***text***`) */ @@ -119,7 +127,10 @@ const PRIORITY = { // Built-in handlers with their option keys and priorities const builtInHandlers: Array<{ handler: RemendHandler; - optionKey: keyof Omit; + optionKey: keyof Omit< + RemendOptions, + "handlers" | "linkMode" | "literalRanges" + >; earlyReturn?: (result: string) => boolean; }> = [ { @@ -278,14 +289,18 @@ const getEnabledBuiltInHandlers = ( }; // Parses markdown text and removes incomplete tokens to prevent partial rendering -const remend = (text: string, options?: RemendOptions): string => { +const repair = (text: string, options?: RemendOptions): string => { if (!text || typeof text !== "string") { return text; } // Remove trailing whitespace if it's not a double space let result = - text.endsWith(" ") && !text.endsWith(" ") ? text.slice(0, -1) : text; + text.endsWith(" ") && + !text.endsWith(" ") && + getScan(text).regions[text.length - 1] !== REGION.LITERAL + ? text.slice(0, -1) + : text; // Get enabled built-in handlers const enabledBuiltIns = getEnabledBuiltInHandlers(options); @@ -316,7 +331,11 @@ const remend = (text: string, options?: RemendOptions): string => { // A handler that removes a trailing fragment can expose a trailing space // (e.g. dropping an incomplete image). Strip it the same way the input // was stripped, so healed output re-heals to itself. - if (result.endsWith(" ") && !result.endsWith(" ")) { + if ( + result.endsWith(" ") && + !result.endsWith(" ") && + getScan(result).regions[result.length - 1] !== REGION.LITERAL + ) { return result.slice(0, -1); } @@ -326,4 +345,7 @@ const remend = (text: string, options?: RemendOptions): string => { } }; +const remend = (text: string, options?: RemendOptions): string => + withLiteralRanges(options?.literalRanges, () => repair(text, options)); + export default remend; diff --git a/packages/remend/src/link-image-handler.ts b/packages/remend/src/link-image-handler.ts index 66cabd0f..608a4966 100644 --- a/packages/remend/src/link-image-handler.ts +++ b/packages/remend/src/link-image-handler.ts @@ -1,3 +1,4 @@ +import { getScan, REGION } from "./scan"; import { isInsideCodeBlock } from "./code-block-utils"; import { findMatchingClosingBracket, @@ -18,8 +19,11 @@ const handleIncompleteUrl = ( lastParenIndex: number, linkMode: LinkMode ): string | null => { - const afterParen = text.substring(lastParenIndex + 2); - if (afterParen.includes(")")) { + let close = text.indexOf(")", lastParenIndex + 2); + while (close !== -1 && getScan(text).regions[close] === REGION.LITERAL) { + close = text.indexOf(")", close + 1); + } + if (close !== -1) { return null; } @@ -93,8 +97,7 @@ const handleIncompleteText = ( const openIndex = isImage ? i - 1 : i; // Check if we have a closing bracket after this - const afterOpen = text.substring(i + 1); - if (!afterOpen.includes("]")) { + if (findMatchingClosingBracket(text, i) === -1) { // This is an incomplete link/image const beforeLink = text.substring(0, openIndex); @@ -148,7 +151,14 @@ const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { // We need to handle nested brackets in the link text // Start from the end and look for ]( pattern - const lastParenIndex = text.lastIndexOf("]("); + let lastParenIndex = text.lastIndexOf("]("); + while ( + lastParenIndex !== -1 && + getScan(text).regions[lastParenIndex] === REGION.LITERAL + ) { + lastParenIndex = + lastParenIndex === 0 ? -1 : text.lastIndexOf("](", lastParenIndex - 1); + } if (lastParenIndex !== -1 && !isInsideCodeBlock(text, lastParenIndex)) { const result = handleIncompleteUrl(text, lastParenIndex, linkMode); if (result !== null) { diff --git a/packages/remend/src/patterns.ts b/packages/remend/src/patterns.ts index 307eb87e..deda4386 100644 --- a/packages/remend/src/patterns.ts +++ b/packages/remend/src/patterns.ts @@ -1,3 +1,4 @@ +import { getScan, REGION } from "./scan"; /** * An end-anchored pattern whose delimiter may follow the opening marker only * as the text's final character. @@ -6,6 +7,7 @@ export interface TrailingPattern { delimiter: string; markerLength: number; regex: RegExp; + trailingDelimiter?: "optional" | "required"; } // A match's marker ends at or after the last delimiter before the final @@ -13,14 +15,62 @@ export interface TrailingPattern { // there finds the same match without scanning the rest of the document. export const matchTrailing = ( text: string, - { delimiter, markerLength, regex }: TrailingPattern + { delimiter, markerLength, regex, trailingDelimiter }: TrailingPattern ): RegExpMatchArray | null => { - const last = text.lastIndexOf(delimiter, text.length - 2); - const markerEnd = last === -1 ? text.length - 1 : last; - return text.slice(Math.max(0, markerEnd - markerLength + 1)).match(regex); + const scan = getScan(text); + if (!scan.regions.includes(REGION.LITERAL)) { + const last = text.lastIndexOf(delimiter, text.length - 2); + const markerEnd = last === -1 ? text.length - 1 : last; + const offset = Math.max(0, markerEnd - markerLength + 1); + const match = text.slice(offset).match(regex); + if (match) match.index = offset + (match.index ?? 0); + return match; + } + // Opaque token contents cannot participate in marker recognition. Scan the + // original source with the existing region classification, without a masked + // source string or independently repaired fragments. + let last = text.length - 2; + while ( + last >= 0 && + (text[last] !== delimiter || scan.regions[last] === REGION.LITERAL) + ) + last--; + const start = Math.max( + 0, + (last < 0 ? text.length - 1 : last) - markerLength + 1 + ); + const marker = delimiter.repeat(markerLength); + if ( + text.slice(start, start + markerLength) !== marker || + scan.regions[start] === REGION.LITERAL + ) + return null; + let trailing = false; + for (let i = start + markerLength; i < text.length; i++) { + if (scan.regions[i] === REGION.LITERAL || text[i] !== delimiter) continue; + if (!trailingDelimiter || i !== text.length - 1) return null; + trailing = true; + } + if ( + trailingDelimiter === "required" && + (!trailing || start + markerLength >= text.length - 1) + ) + return null; + const match = [ + text.slice(start), + marker, + text.slice( + start + markerLength, + trailingDelimiter === "required" ? -1 : undefined + ), + ] as RegExpMatchArray; + match.index = start; + match.input = text; + return match; }; export const boldPattern: TrailingPattern = { + trailingDelimiter: "optional", delimiter: "*", markerLength: 2, regex: /(\*\*)([^*]*\*?)$/, @@ -51,11 +101,13 @@ export const strikethroughPattern: TrailingPattern = { regex: /(~~)([^~]*?)$/, }; export const halfCompleteUnderscorePattern: TrailingPattern = { + trailingDelimiter: "required", delimiter: "_", markerLength: 2, regex: /(__)([^_]+)_$/, }; export const halfCompleteTildePattern: TrailingPattern = { + trailingDelimiter: "required", delimiter: "~", markerLength: 2, regex: /(~~)([^~]+)~$/, diff --git a/packages/remend/src/scan.ts b/packages/remend/src/scan.ts index 1bb9f74f..673ea48b 100644 --- a/packages/remend/src/scan.ts +++ b/packages/remend/src/scan.ts @@ -32,6 +32,8 @@ export const REGION = { PROSE: 0, + /** Parser-owned literal source: opaque to all repair delimiters. */ + LITERAL: 6, /** The ``` or ~~~ run that opens or closes a fence */ FENCE_MARKER: 1, /** The info string on a fence opener line. Neither prose nor code body. */ @@ -220,6 +222,7 @@ const openFenceAt = ( const contentEnd = lineEnd > lineStart && text[lineEnd - 1] === "\r" ? lineEnd - 1 : lineEnd; const prefix = skipContainerPrefix(text, lineStart, contentEnd); + if (regions[prefix.contentStart] === REGION.LITERAL) return null; const opener = text .slice(prefix.contentStart, contentEnd) .match(FENCE_RUN_PATTERN); @@ -406,7 +409,15 @@ const paintSpans = (text: string, regions: Uint8Array): OpenSpan | null => { const scanText = (text: string): TextScan => { const regions = new Uint8Array(text.length); + const opaque = literalRanges?.(text) ?? []; + for (const { start, end } of opaque) { + regions.fill(REGION.LITERAL, start, end); + } const openFence = paintFences(text, regions); + // Conservative fence painting must never take ownership of literal tokens. + for (const { start, end } of opaque) { + regions.fill(REGION.LITERAL, start, end); + } const openSpan = paintSpans(text, regions); return { text, @@ -423,6 +434,27 @@ const scanText = (text: string): TextScan => { // string, and remend's handler chain passes each handler's output to the // next, so a single-entry cache gives O(1) queries within a handler while // staying O(n) per handler overall. +export type LiteralRanges = ( + text: string +) => ReadonlyArray<{ start: number; end: number }>; +let literalRanges: LiteralRanges | undefined; + +/** Scope parser-owned opaque regions to one synchronous repair call. */ +export function withLiteralRanges( + ranges: LiteralRanges | undefined, + run: () => T +): T { + const previous = literalRanges; + literalRanges = ranges; + clearScanCache(); + try { + return run(); + } finally { + literalRanges = previous; + clearScanCache(); + } +} + let cachedText: string | null = null; let cachedScan: TextScan | null = null; @@ -464,6 +496,7 @@ export const isFenceAt = (scan: TextScan, position: number): boolean => { } const region = scan.regions[position]; return ( + region === REGION.LITERAL || region === REGION.FENCE_MARKER || region === REGION.FENCE_INFO || region === REGION.FENCE_BODY @@ -471,7 +504,8 @@ export const isFenceAt = (scan: TextScan, position: number): boolean => { }; export const isCompleteSpanAt = (scan: TextScan, position: number): boolean => - scan.regions[position] === REGION.CODE_SPAN; + scan.regions[position] === REGION.CODE_SPAN || + scan.regions[position] === REGION.LITERAL; /** Counts non-overlapping double-character pairs (**, ~~, $$) in prose */ export const countDoublePairs = (text: string, char: string): number => { diff --git a/packages/remend/src/strikethrough-handler.ts b/packages/remend/src/strikethrough-handler.ts index e981f83c..f272be96 100644 --- a/packages/remend/src/strikethrough-handler.ts +++ b/packages/remend/src/strikethrough-handler.ts @@ -32,7 +32,7 @@ export const handleIncompleteStrikethrough = (text: string): string => { } // Don't close if the marker is inside an inline code span or fenced code block - const markerIndex = text.lastIndexOf(strikethroughMatch[1]); + const markerIndex = strikethroughMatch.index!; if ( isInsideCodeBlock(text, markerIndex) || isWithinCompleteInlineCode(text, markerIndex) @@ -49,7 +49,7 @@ export const handleIncompleteStrikethrough = (text: string): string => { const halfCompleteMatch = matchTrailing(text, halfCompleteTildePattern); if (halfCompleteMatch) { // Don't close if the marker is inside an inline code span or fenced code block - const markerIndex = text.lastIndexOf(halfCompleteMatch[0].slice(0, 2)); + const markerIndex = halfCompleteMatch.index!; if ( isInsideCodeBlock(text, markerIndex) || isWithinCompleteInlineCode(text, markerIndex) diff --git a/packages/remend/src/utils.ts b/packages/remend/src/utils.ts index 06291fd0..63cfaa0c 100644 --- a/packages/remend/src/utils.ts +++ b/packages/remend/src/utils.ts @@ -1,5 +1,12 @@ import { letterNumberUnderscorePattern } from "./patterns"; -import { getScan, inHtmlTagAt, inLinkUrlAt, inMathAt, isFenceAt } from "./scan"; +import { + getScan, + inHtmlTagAt, + inLinkUrlAt, + inMathAt, + isFenceAt, + REGION, +} from "./scan"; // OPTIMIZATION: Precompute which characters are word characters // Using ASCII fast path before falling back to Unicode regex @@ -32,6 +39,7 @@ export const findMatchingOpeningBracket = ( ): number => { let depth = 1; for (let i = closeIndex - 1; i >= 0; i -= 1) { + if (getScan(text).regions[i] === REGION.LITERAL) continue; if (text[i] === "]") { depth += 1; } else if (text[i] === "[") { @@ -52,6 +60,7 @@ export const findMatchingClosingBracket = ( ): number => { let depth = 1; for (let i = openIndex + 1; i < text.length; i += 1) { + if (getScan(text).regions[i] === REGION.LITERAL) continue; if (text[i] === "[") { depth += 1; } else if (text[i] === "]") { From 1b0ac024f9204f2b5438bda3cd39d3b7be8c22b0 Mon Sep 17 00:00:00 2001 From: David Fant Date: Mon, 26 Oct 2026 19:06:04 -0700 Subject: [PATCH 2/4] test(remend): cover literal range scope isolation --- .../__tests__/literal-ranges-scope.test.ts | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 packages/remend/__tests__/literal-ranges-scope.test.ts diff --git a/packages/remend/__tests__/literal-ranges-scope.test.ts b/packages/remend/__tests__/literal-ranges-scope.test.ts new file mode 100644 index 00000000..c2832372 --- /dev/null +++ b/packages/remend/__tests__/literal-ranges-scope.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, test } from "vitest"; +import remend from "../src/index"; +const all = (source: string) => [{ start: 0, end: source.length }]; +const value = "20~25"; + +describe("Independent opaque-range dynamic scope acceptance", () => { + test("unprotected nested invocation does not inherit outer opaque ranges or poison outer cache", () => { + let nested: string | undefined; + const output = remend(value, { + literalRanges: all, + handlers: [ + { + name: "independent-nested-unprotected", + priority: -10, + handle: (source) => { + nested = remend(value); + return source; + }, + }, + ], + }); + expect(nested).toBe("20\\~25"); + expect(output).toBe(value); + expect(remend(value)).toBe("20\\~25"); + }); + test("protected nested invocation does not leak ranges into unprotected outer repair", () => { + let nested: string | undefined; + const output = remend(value, { + handlers: [ + { + name: "independent-nested-protected", + priority: -10, + handle: (source) => { + nested = remend(value, { literalRanges: all }); + return source; + }, + }, + ], + }); + expect(nested).toBe(value); + expect(output).toBe("20\\~25"); + }); + test("nested call from range callback restores same-string scanner context", () => { + let nested: string | undefined; + const output = remend(value, { + literalRanges: (source) => { + nested = remend(value); + return all(source); + }, + }); + expect(nested).toBe("20\\~25"); + expect(output).toBe(value); + }); + test("throwing user handler clears opaque scope and cached scan for subsequent calls", () => { + expect(() => + remend(value, { + literalRanges: all, + handlers: [ + { + name: "independent-throwing-handler", + priority: 100, + handle: () => { + throw Error("expected scoped failure"); + }, + }, + ], + }) + ).toThrow("expected scoped failure"); + expect(remend(value)).toBe("20\\~25"); + }); + test("throwing range callback clears scope even when scan occurs before handler try/finally", () => { + expect(() => + remend(value + " ", { + literalRanges: () => { + throw Error("expected range failure"); + }, + }) + ).toThrow("expected range failure"); + expect(remend(value)).toBe("20\\~25"); + }); + test("caught inner callback failure restores outer protected scope", () => { + const output = remend(value, { + literalRanges: all, + handlers: [ + { + name: "independent-nested-caught-failure", + priority: -10, + handle: (source) => { + expect(() => + remend(value + " ", { + literalRanges: () => { + throw Error("expected inner failure"); + }, + }) + ).toThrow("expected inner failure"); + return source; + }, + }, + ], + }); + expect(output).toBe(value); + expect(remend(value)).toBe("20\\~25"); + }); +}); From 75896c53753f9e02c48cb46ec952c920e4901a1a Mon Sep 17 00:00:00 2001 From: David Fant Date: Mon, 26 Oct 2026 19:07:34 -0700 Subject: [PATCH 3/4] docs(remend): clarify caller-owned literal range parsing --- packages/remend/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/remend/README.md b/packages/remend/README.md index f3481248..59d442a7 100644 --- a/packages/remend/README.md +++ b/packages/remend/README.md @@ -216,13 +216,12 @@ Remend is built for high-performance streaming scenarios: For more info, see the [documentation](https://streamdown.ai/docs/termination). - ### Parser-owned literal ranges `literalRanges` optionally supplies opaque source intervals to the existing repair scanner. Built-in handlers keep the full Markdown context but do not interpret or -edit those literal bodies. Streamdown supplies ranges from its registered-container -micromark grammar, not a second tag matcher. +edit those literal bodies. A caller can supply ranges from its parser; Remend does +not interpret container syntax. ```ts remend(source, { From ba8b9e7e7061e6b78b0f9f9bcb93daea0ae13e69 Mon Sep 17 00:00:00 2001 From: David Fant Date: Mon, 26 Oct 2026 19:19:39 -0700 Subject: [PATCH 4/4] fix(remend): respect opaque boundaries in repair edits --- .../__tests__/literal-boundaries.test.ts | 98 +++++++++++++++++++ .../remend/src/comparison-operator-handler.ts | 2 +- packages/remend/src/emphasis-handlers.ts | 17 +++- packages/remend/src/html-tag-handler.ts | 13 ++- packages/remend/src/katex-handler.ts | 6 +- packages/remend/src/link-image-handler.ts | 16 ++- packages/remend/src/setext-heading-handler.ts | 9 ++ 7 files changed, 150 insertions(+), 11 deletions(-) create mode 100644 packages/remend/__tests__/literal-boundaries.test.ts diff --git a/packages/remend/__tests__/literal-boundaries.test.ts b/packages/remend/__tests__/literal-boundaries.test.ts new file mode 100644 index 00000000..b43e7edc --- /dev/null +++ b/packages/remend/__tests__/literal-boundaries.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest"; +import remend from "../src/index"; + +const protect = (payload: string) => (source: string) => { + const start = source.indexOf(payload); + return start < 0 ? [] : [{ start, end: start + payload.length }]; +}; + +describe("opaque edit boundaries", () => { + it.each([ + "-", + "--", + "=", + "==", + ])("does not repair a protected setext line %s", (marker) => { + const source = `opaque\n${marker}`; + expect(remend(source, { literalRanges: protect(source) })).toBe(source); + }); + it("does not trim protected whitespace before an incomplete HTML tag", () => { + const payload = "opaque \n "; + expect(remend(`${payload} { + const payload = "opaque payload"; + const source = `[label](url ${payload} tail`; + const prefix = + linkMode === "protocol" ? "[label](streamdown:incomplete-link)" : "label"; + expect(remend(source, { literalRanges: protect(payload), linkMode })).toBe( + `${prefix}${payload} tail` + ); + }); + it("inserts an underscore outside protected trailing newlines", () => { + const payload = "opaque\n\n"; + const source = `_before ${payload}`; + expect(remend(source, { literalRanges: protect(payload) })).toBe( + `${source}_` + ); + }); + it.each([ + "protocol", + "text-only", + ] as const)("keeps an opaque tail after an incomplete image URL in %s mode", (linkMode) => { + const payload = "opaque payload"; + expect( + remend(`![alt](url ${payload} tail`, { + literalRanges: protect(payload), + linkMode, + }) + ).toBe(`![alt](streamdown:incomplete-image)${payload} tail`); + }); + it("does not let an HTML opener before a literal swallow it", () => { + const payload = "opaque"; + const source = ` { + const payload = "opaque "; + expect( + remend(` { + const payload = "opaque**"; + const source = `**before _${payload}`; + expect(remend(source, { literalRanges: protect(payload) })).toBe( + `${source}_**` + ); + }); + it("does not escape a protected comparison after an unprotected list prefix", () => { + const payload = "> 25"; + const source = `- ${payload}`; + expect(remend(source, { literalRanges: protect(payload) })).toBe(source); + }); + it.each([ + "\n", + "\n\n", + ])("places underscore completion before only unprotected trailing newlines %j", (ending) => { + const payload = "opaque\n"; + const source = `_before ${payload}${ending}`; + expect(remend(source, { literalRanges: protect(payload) })).toBe( + `_before ${payload}_${ending}` + ); + }); + it.each([ + ["**before ", "opaque*", "**"], + ["$$before ", "opaque$", "$$"], + ])("does not count a literal partial closing marker for %s", (prefix, payload, closing) => { + const source = `${prefix}${payload}`; + expect(remend(source, { literalRanges: protect(payload) })).toBe( + `${source}${closing}` + ); + }); +}); diff --git a/packages/remend/src/comparison-operator-handler.ts b/packages/remend/src/comparison-operator-handler.ts index 93fc1a5b..7d961fd8 100644 --- a/packages/remend/src/comparison-operator-handler.ts +++ b/packages/remend/src/comparison-operator-handler.ts @@ -23,7 +23,7 @@ export const handleComparisonOperators = (text: string): string => { LIST_COMPARISON_PATTERN, (match, prefix, suffix, offset) => { // Don't escape inside code blocks - if (isInsideCodeBlock(text, offset)) { + if (isInsideCodeBlock(text, offset + prefix.length)) { return match; } diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 33229608..8820cd35 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -416,7 +416,10 @@ export const handleIncompleteBold = (text: string): string => { if (asteriskPairs % 2 === 1) { // Check for half-complete closing marker: **content* should become **content** // The trailing * is the first char of the closing ** being streamed - if (contentAfterMarker.endsWith("*")) { + if ( + contentAfterMarker.endsWith("*") && + getScan(text).regions[text.length - 1] !== REGION.LITERAL + ) { return `${text}*`; } return `${text}**`; @@ -632,7 +635,12 @@ const insertClosingUnderscore = (text: string): string => { // If text ends with newline(s), insert underscore before them // Use string methods instead of regex to avoid ReDoS vulnerability let endIndex = text.length; - while (endIndex > 0 && text[endIndex - 1] === "\n") { + const scan = getScan(text); + while ( + endIndex > 0 && + text[endIndex - 1] === "\n" && + scan.regions[endIndex - 1] !== REGION.LITERAL + ) { endIndex -= 1; } if (endIndex < text.length) { @@ -645,7 +653,10 @@ const insertClosingUnderscore = (text: string): string => { // Helper to handle trailing ** for proper nesting of _ and ** markers const handleTrailingAsterisksForUnderscore = (text: string): string | null => { - if (!text.endsWith("**")) { + if ( + !text.endsWith("**") || + getScan(text).regions.indexOf(REGION.LITERAL, text.length - 2) !== -1 + ) { return null; } diff --git a/packages/remend/src/html-tag-handler.ts b/packages/remend/src/html-tag-handler.ts index c837e4b5..df9853ce 100644 --- a/packages/remend/src/html-tag-handler.ts +++ b/packages/remend/src/html-tag-handler.ts @@ -1,4 +1,5 @@ import { isInsideCodeBlock } from "./code-block-utils"; +import { getScan, REGION } from "./scan"; import { isWithinMathBlock } from "./utils"; // Matches an incomplete HTML tag at the end of the string. @@ -42,8 +43,16 @@ export const handleIncompleteHtmlTag = (text: string): string => { continue; } - // Strip the incomplete tag and any trailing whitespace before it - return text.substring(0, index).trimEnd(); + const scan = getScan(text); + // An incomplete tag cannot consume an opaque region later in the source. + if (scan.regions.indexOf(REGION.LITERAL, index) !== -1) { + continue; + } + + // Trim only prose whitespace: the preceding literal owns its final bytes. + const trimmedEnd = text.substring(0, index).trimEnd().length; + const literalEnd = scan.regions.lastIndexOf(REGION.LITERAL, index - 1) + 1; + return text.substring(0, Math.max(trimmedEnd, literalEnd)); } return text; diff --git a/packages/remend/src/katex-handler.ts b/packages/remend/src/katex-handler.ts index c274fd70..f4b87663 100644 --- a/packages/remend/src/katex-handler.ts +++ b/packages/remend/src/katex-handler.ts @@ -33,7 +33,11 @@ const countSingleDollars = (text: string): number => { const addClosingKatex = (text: string): string => { // If the text already ends with a partial closing $ (but not $$), // just append one more $ to complete the $$ marker. - if (text.endsWith("$") && !text.endsWith("$$")) { + if ( + text.endsWith("$") && + !text.endsWith("$$") && + getScan(text).regions[text.length - 1] !== REGION.LITERAL + ) { return `${text}$`; } diff --git a/packages/remend/src/link-image-handler.ts b/packages/remend/src/link-image-handler.ts index 608a4966..50f35a78 100644 --- a/packages/remend/src/link-image-handler.ts +++ b/packages/remend/src/link-image-handler.ts @@ -27,6 +27,13 @@ const handleIncompleteUrl = ( return null; } + // URL repair ends at the first opaque boundary, not at the end of the document. + const literalStart = getScan(text).regions.indexOf( + REGION.LITERAL, + lastParenIndex + 2 + ); + const afterUrl = literalStart === -1 ? "" : text.substring(literalStart); + // We have an incomplete URL like [text](partial-url // Now find the matching opening bracket for the ] before ( const openBracketIndex = findMatchingOpeningBracket(text, lastParenIndex); @@ -47,14 +54,14 @@ const handleIncompleteUrl = ( if (isImage) { // For images with incomplete URLs, replace with placeholder marker - return `${beforeLink}![${altOrLinkText}](${INCOMPLETE_IMAGE_PLACEHOLDER})`; + return `${beforeLink}![${altOrLinkText}](${INCOMPLETE_IMAGE_PLACEHOLDER})${afterUrl}`; } // For links with incomplete URLs, handle based on linkMode if (linkMode === "text-only") { - return `${beforeLink}${altOrLinkText}`; + return `${beforeLink}${altOrLinkText}${afterUrl}`; } - return `${beforeLink}[${altOrLinkText}](streamdown:incomplete-link)`; + return `${beforeLink}[${altOrLinkText}](streamdown:incomplete-link)${afterUrl}`; }; // Helper to find the first incomplete [ (for text-only mode) @@ -154,7 +161,8 @@ const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { let lastParenIndex = text.lastIndexOf("]("); while ( lastParenIndex !== -1 && - getScan(text).regions[lastParenIndex] === REGION.LITERAL + (getScan(text).regions[lastParenIndex] === REGION.LITERAL || + getScan(text).regions[lastParenIndex + 1] === REGION.LITERAL) ) { lastParenIndex = lastParenIndex === 0 ? -1 : text.lastIndexOf("](", lastParenIndex - 1); diff --git a/packages/remend/src/setext-heading-handler.ts b/packages/remend/src/setext-heading-handler.ts index 3ae0d5c7..2f91070d 100644 --- a/packages/remend/src/setext-heading-handler.ts +++ b/packages/remend/src/setext-heading-handler.ts @@ -1,3 +1,5 @@ +import { getScan, REGION } from "./scan"; + // Handles incomplete setext heading underlines during streaming // Setext headings use --- or === on the line below text to create headings // During streaming, partial list items (like "-") can be misinterpreted as setext heading underlines @@ -25,6 +27,13 @@ export const handleIncompleteSetextHeading = (text: string): string => { return text; } + // An opaque final line is not Markdown, even when its bytes look like an underline. + if ( + getScan(text).regions.indexOf(REGION.LITERAL, lastNewlineIndex + 1) !== -1 + ) { + return text; + } + const lastLine = text.substring(lastNewlineIndex + 1); const previousContent = text.substring(0, lastNewlineIndex);