feat(remend): protect parser-owned literal ranges during repair

.changeset/protected-literal-repair.md

@@ -0,0 +1,8 @@
+---
+"remend": patch
+---
+
+Allow syntax extensions to supply opaque source ranges through `literalRanges`.
+Incomplete-Markdown repair preserves those ranges while retaining full-document
+formatting context. Range callbacks use the current source after each edit, and
+nested calls and exceptions restore the caller's scanner scope.

packages/remend/README.md

@@ -215,3 +215,23 @@ Remend is built for high-performance streaming scenarios:
 - Early returns for common cases
 
 For more info, see the [documentation](https://streamdown.ai/docs/termination).
+
+### Parser-owned literal ranges
+
+`literalRanges` optionally supplies opaque source intervals to the existing repair
+scanner. Built-in handlers keep the full Markdown context but do not interpret or
+edit those literal bodies. A caller can supply ranges from its parser; Remend does
+not interpret container syntax.
+
+```ts
+remend(source, {
+  literalRanges: currentSource => parserLiteralRanges(currentSource),
+});
+```
+
+The callback returns UTF-16 `{ start, end }` offsets, with an exclusive end, for
+sorted, non-overlapping intervals. It is cached with the current source scan and
+called again when an earlier handler changes the source. Scope is synchronous and
+restored after nested calls or exceptions. There is no source encoding or
+fragment-by-fragment repair. User-defined custom handlers still receive the entire
+string and remain responsible for respecting literal source themselves.

packages/remend/__tests__/literal-boundaries.test.ts

@@ -0,0 +1,98 @@
+import { describe, expect, it } from "vitest";
+import remend from "../src/index";
+
+const protect = (payload: string) => (source: string) => {
+  const start = source.indexOf(payload);
+  return start < 0 ? [] : [{ start, end: start + payload.length }];
+};
+
+describe("opaque edit boundaries", () => {
+  it.each([
+    "-",
+    "--",
+    "=",
+    "==",
+  ])("does not repair a protected setext line %s", (marker) => {
+    const source = `opaque\n${marker}`;
+    expect(remend(source, { literalRanges: protect(source) })).toBe(source);
+  });
+  it("does not trim protected whitespace before an incomplete HTML tag", () => {
+    const payload = "opaque \n ";
+    expect(remend(`${payload}<tag`, { literalRanges: protect(payload) })).toBe(
+      payload
+    );
+  });
+  it.each([
+    "protocol",
+    "text-only",
+  ] as const)("keeps opaque payload and following prose after an incomplete URL in %s mode", (linkMode) => {
+    const payload = "opaque payload";
+    const source = `[label](url ${payload} tail`;
+    const prefix =
+      linkMode === "protocol" ? "[label](streamdown:incomplete-link)" : "label";
+    expect(remend(source, { literalRanges: protect(payload), linkMode })).toBe(
+      `${prefix}${payload} tail`
+    );
+  });
+  it("inserts an underscore outside protected trailing newlines", () => {
+    const payload = "opaque\n\n";
+    const source = `_before ${payload}`;
+    expect(remend(source, { literalRanges: protect(payload) })).toBe(
+      `${source}_`
+    );
+  });
+  it.each([
+    "protocol",
+    "text-only",
+  ] as const)("keeps an opaque tail after an incomplete image URL in %s mode", (linkMode) => {
+    const payload = "opaque payload";
+    expect(
+      remend(`![alt](url ${payload} tail`, {
+        literalRanges: protect(payload),
+        linkMode,
+      })
+    ).toBe(`![alt](streamdown:incomplete-image)${payload} tail`);
+  });
+  it("does not let an HTML opener before a literal swallow it", () => {
+    const payload = "opaque";
+    const source = `<tag ${payload} tail`;
+    expect(remend(source, { literalRanges: protect(payload) })).toBe(source);
+  });
+  it("can still remove a later incomplete tag after an opaque region", () => {
+    const payload = "opaque ";
+    expect(
+      remend(`<tag ${payload}<other`, { literalRanges: protect(payload) })
+    ).toBe(`<tag ${payload}`);
+  });
+  it("does not move an underscore into a literal trailing asterisk pair", () => {
+    const payload = "opaque**";
+    const source = `**before _${payload}`;
+    expect(remend(source, { literalRanges: protect(payload) })).toBe(
+      `${source}_**`
+    );
+  });
+  it("does not escape a protected comparison after an unprotected list prefix", () => {
+    const payload = "> 25";
+    const source = `- ${payload}`;
+    expect(remend(source, { literalRanges: protect(payload) })).toBe(source);
+  });
+  it.each([
+    "\n",
+    "\n\n",
+  ])("places underscore completion before only unprotected trailing newlines %j", (ending) => {
+    const payload = "opaque\n";
+    const source = `_before ${payload}${ending}`;
+    expect(remend(source, { literalRanges: protect(payload) })).toBe(
+      `_before ${payload}_${ending}`
+    );
+  });
+  it.each([
+    ["**before ", "opaque*", "**"],
+    ["$$before ", "opaque$", "$$"],
+  ])("does not count a literal partial closing marker for %s", (prefix, payload, closing) => {
+    const source = `${prefix}${payload}`;
+    expect(remend(source, { literalRanges: protect(payload) })).toBe(
+      `${source}${closing}`
+    );
+  });
+});

packages/remend/__tests__/literal-ranges-scope.test.ts

@@ -0,0 +1,104 @@
+import { describe, expect, test } from "vitest";
+import remend from "../src/index";
+const all = (source: string) => [{ start: 0, end: source.length }];
+const value = "20~25";
+
+describe("Independent opaque-range dynamic scope acceptance", () => {
+  test("unprotected nested invocation does not inherit outer opaque ranges or poison outer cache", () => {
+    let nested: string | undefined;
+    const output = remend(value, {
+      literalRanges: all,
+      handlers: [
+        {
+          name: "independent-nested-unprotected",
+          priority: -10,
+          handle: (source) => {
+            nested = remend(value);
+            return source;
+          },
+        },
+      ],
+    });
+    expect(nested).toBe("20\\~25");
+    expect(output).toBe(value);
+    expect(remend(value)).toBe("20\\~25");
+  });
+  test("protected nested invocation does not leak ranges into unprotected outer repair", () => {
+    let nested: string | undefined;
+    const output = remend(value, {
+      handlers: [
+        {
+          name: "independent-nested-protected",
+          priority: -10,
+          handle: (source) => {
+            nested = remend(value, { literalRanges: all });
+            return source;
+          },
+        },
+      ],
+    });
+    expect(nested).toBe(value);
+    expect(output).toBe("20\\~25");
+  });
+  test("nested call from range callback restores same-string scanner context", () => {
+    let nested: string | undefined;
+    const output = remend(value, {
+      literalRanges: (source) => {
+        nested = remend(value);
+        return all(source);
+      },
+    });
+    expect(nested).toBe("20\\~25");
+    expect(output).toBe(value);
+  });
+  test("throwing user handler clears opaque scope and cached scan for subsequent calls", () => {
+    expect(() =>
+      remend(value, {
+        literalRanges: all,
+        handlers: [
+          {
+            name: "independent-throwing-handler",
+            priority: 100,
+            handle: () => {
+              throw Error("expected scoped failure");
+            },
+          },
+        ],
+      })
+    ).toThrow("expected scoped failure");
+    expect(remend(value)).toBe("20\\~25");
+  });
+  test("throwing range callback clears scope even when scan occurs before handler try/finally", () => {
+    expect(() =>
+      remend(value + " ", {
+        literalRanges: () => {
+          throw Error("expected range failure");
+        },
+      })
+    ).toThrow("expected range failure");
+    expect(remend(value)).toBe("20\\~25");
+  });
+  test("caught inner callback failure restores outer protected scope", () => {
+    const output = remend(value, {
+      literalRanges: all,
+      handlers: [
+        {
+          name: "independent-nested-caught-failure",
+          priority: -10,
+          handle: (source) => {
+            expect(() =>
+              remend(value + " ", {
+                literalRanges: () => {
+                  throw Error("expected inner failure");
+                },
+              })
+            ).toThrow("expected inner failure");
+            return source;
+          },
+        },
+      ],
+    });
+    expect(output).toBe(value);
+    expect(remend(value)).toBe("20\\~25");
+  });
+});

packages/remend/__tests__/literal-ranges.test.ts

@@ -0,0 +1,78 @@
+import { describe, expect, it } from "vitest";
+import remend from "../src/index";
+
+function literalRanges(text: string) {
+  const start = text.indexOf("<literal>");
+  const close = text.indexOf("</literal>", start);
+  return start < 0
+    ? []
+    : [{ start, end: close < 0 ? text.length : close + "</literal>".length }];
+}
+describe("parser-owned opaque ranges", () => {
+  it.each([
+    "20~25",
+    "[unclosed](https://example.com",
+    "**raw",
+    "`raw",
+    "$$math",
+    "\n- > 25",
+    "\n```\ncode",
+    "trailing ",
+  ])("never heals literal source %j", (body) => {
+    const source = `<literal>${body}</literal>`;
+    expect(remend(source, { literalRanges })).toBe(source);
+  });
+  it("preserves full surrounding Markdown context", () => {
+    const source = "**Before <literal>**raw</literal> after**";
+    expect(remend(source, { literalRanges })).toBe(source);
+    expect(
+      remend("**Before <literal>**raw</literal> after", { literalRanges })
+    ).toBe("**Before <literal>**raw</literal> after**");
+  });
+  it("ignores literal brackets while healing a crossing link", () => {
+    const source = "[Before <literal>][fake](raw</literal> after";
+    expect(remend(source, { literalRanges })).toBe(
+      source + "](streamdown:incomplete-link)"
+    );
+  });
+  it("recomputes offsets when earlier repairs insert characters", () => {
+    expect(
+      remend("20~25 <literal>20~25 **raw</literal>", { literalRanges })
+    ).toBe("20\\~25 <literal>20~25 **raw</literal>");
+  });
+  it("does not trim whitespace owned by an unfinished literal", () => {
+    expect(remend("<literal>raw ", { literalRanges })).toBe("<literal>raw ");
+  });
+  it("restores scoped ranges after nested calls", () => {
+    const result = remend("<literal>20~25</literal>", {
+      literalRanges,
+      handlers: [
+        {
+          name: "nested",
+          handle: (text) => {
+            expect(remend("20~25")).toBe("20\\~25");
+            return text;
+          },
+        },
+      ],
+    });
+    expect(result).toBe("<literal>20~25</literal>");
+    expect(remend("20~25")).toBe("20\\~25");
+  });
+  it("clears scoped ranges after a handler throws", () => {
+    expect(() =>
+      remend("<literal>raw</literal>", {
+        literalRanges,
+        handlers: [
+          {
+            name: "throw",
+            handle: () => {
+              throw new Error("test");
+            },
+          },
+        ],
+      })
+    ).toThrow("test");
+    expect(remend("20~25")).toBe("20\\~25");
+  });
+});

packages/remend/src/comparison-operator-handler.ts

@@ -23,7 +23,7 @@ export const handleComparisonOperators = (text: string): string => {
     LIST_COMPARISON_PATTERN,
     (match, prefix, suffix, offset) => {
       // Don't escape inside code blocks
-      if (isInsideCodeBlock(text, offset)) {
+      if (isInsideCodeBlock(text, offset + prefix.length)) {
         return match;
       }
 

packages/remend/src/emphasis-handlers.ts

@@ -398,7 +398,7 @@ export const handleIncompleteBold = (text: string): string => {
   }
 
   const contentAfterMarker = boldMatch[2];
-  const markerIndex = text.lastIndexOf(boldMatch[1]);
+  const markerIndex = boldMatch.index!;
 
   // Check if the bold marker is within a code block (fenced or inline)
   if (
@@ -416,7 +416,10 @@ export const handleIncompleteBold = (text: string): string => {
   if (asteriskPairs % 2 === 1) {
     // Check for half-complete closing marker: **content* should become **content**
     // The trailing * is the first char of the closing ** being streamed
-    if (contentAfterMarker.endsWith("*")) {
+    if (
+      contentAfterMarker.endsWith("*") &&
+      getScan(text).regions[text.length - 1] !== REGION.LITERAL
+    ) {
       return `${text}*`;
     }
     return `${text}**`;
@@ -469,7 +472,7 @@ export const handleIncompleteDoubleUnderscoreItalic = (
       halfCompleteUnderscorePattern
     );
     if (halfCompleteMatch) {
-      const markerIndex = text.lastIndexOf(halfCompleteMatch[1]);
+      const markerIndex = halfCompleteMatch.index!;
       if (
         !(
           isInsideCodeBlock(text, markerIndex) ||
@@ -484,7 +487,7 @@ export const handleIncompleteDoubleUnderscoreItalic = (
   }
 
   const contentAfterMarker = italicMatch[2];
-  const markerIndex = text.lastIndexOf(italicMatch[1]);
+  const markerIndex = italicMatch.index!;
 
   // Check if the italic marker is within a code block (fenced or inline)
   if (
@@ -632,7 +635,12 @@ const insertClosingUnderscore = (text: string): string => {
   // If text ends with newline(s), insert underscore before them
   // Use string methods instead of regex to avoid ReDoS vulnerability
   let endIndex = text.length;
-  while (endIndex > 0 && text[endIndex - 1] === "\n") {
+  const scan = getScan(text);
+  while (
+    endIndex > 0 &&
+    text[endIndex - 1] === "\n" &&
+    scan.regions[endIndex - 1] !== REGION.LITERAL
+  ) {
     endIndex -= 1;
   }
   if (endIndex < text.length) {
@@ -645,7 +653,10 @@ const insertClosingUnderscore = (text: string): string => {
 
 // Helper to handle trailing ** for proper nesting of _ and ** markers
 const handleTrailingAsterisksForUnderscore = (text: string): string | null => {
-  if (!text.endsWith("**")) {
+  if (
+    !text.endsWith("**") ||
+    getScan(text).regions.indexOf(REGION.LITERAL, text.length - 2) !== -1
+  ) {
     return null;
   }
 
@@ -770,7 +781,7 @@ export const handleIncompleteBoldItalic = (text: string): string => {
   }
 
   const contentAfterMarker = boldItalicMatch[2];
-  const markerIndex = text.lastIndexOf(boldItalicMatch[1]);
+  const markerIndex = boldItalicMatch.index!;
 
   if (shouldSkipBoldItalicCompletion(text, contentAfterMarker, markerIndex)) {
     return text;