feat(lexer): emit one jsx_text token per JSX text child in .jsx (#61)

src/parser.zig

@@ -904,11 +904,14 @@ pub const Parser = struct {
     /// @returns borrowed_from(self)
     pub fn tokenText(self: *const Parser, index: TokenIndex) []const u8 {
         const tag = self.tags_ptr[index];
-        // Variable-lexeme tokens (identifiers, literals, escaped_keyword, etc.) are the first
-        // 9 enum variants (0..identifier) plus the last 4 (escaped_keyword..jsx_text).
-        // A range check is cheaper than the 80-arm lexeme() switch for these common cases.
+        // Variable-lexeme tokens carry their own source text. They are the first 9
+        // enum variants (0..identifier) and the tail group escaped_keyword/at_sign/
+        // hashbang — a range check is cheaper than the 80-arm lexeme() switch. jsx_text
+        // is also variable-lexeme but sits before eof in the enum (outside the tail
+        // range), so it needs an explicit case; lexeme(.jsx_text) is null, so without
+        // this it would null-deref. (eof/invalid keep their fixed `<eof>`/`<invalid>`.)
         const ti = @intFromEnum(tag);
-        if (ti <= @intFromEnum(TokenTag.identifier) or ti >= @intFromEnum(TokenTag.escaped_keyword)) {
+        if (ti <= @intFromEnum(TokenTag.identifier) or ti >= @intFromEnum(TokenTag.escaped_keyword) or tag == .jsx_text) {
             const start = self.tok_starts_ptr[index];
             const len = self.tok_lens_ptr[index];
             return self.source[start .. start + len];

src/scalar_lexer.zig

@@ -486,6 +486,26 @@ pub const Token = struct {
     has_unicode_escape: bool = false,
 };
 
+/// Top bit of a JSX frame marks an expression container (vs an element body);
+/// the low bits count its nested object/block braces (#61).
+const JSX_EXPR_BIT: u32 = 0x8000_0000;
+
+/// Per-call JSX element-body tracking state (#61). Comptime-conditional: returns a
+/// zero-size struct when text tokenization is off, so the non-JSX / TSX instantiation
+/// of the lexer carries no frame array and no tracking fields at all.
+fn JsxState(comptime enabled: bool) type {
+    if (!enabled) return struct {};
+    return struct {
+        in_text: bool = false,
+        closing: bool = false, // the `<` just seen begins a closing tag `</…`
+        sp: u32 = 0,
+        frame_inline: [64]u32 = undefined,
+        frame_ptr: [*]u32 = undefined,
+        frame_cap: u32 = 64,
+        frame_heap: ?[]u32 = null,
+        text_nl: bool = false, // last jsx_text token contained a line terminator
+    };
+}
 
 /// Tokenize `src` into a `TokenList` using the default options, matching
 /// `Lexer.tokenizeWithLanguage`. `language` selects the TS keyword set and JSX.
@@ -533,6 +553,26 @@ pub fn tokenizeScalarWithOptions(
     src: []const u8,
     language: Language,
     opts: Lex.TokenizeOptions,
+) !TokenList {
+    // JSX text tokenization (#61) needs to know when a `<tag>` opens an element
+    // body. In TSX a `<T>` is ambiguous (JSX element vs generic type arguments vs
+    // generic arrow), and only the parser's speculative parse can disambiguate, so
+    // element-body tracking is restricted to plain JSX (no TS); TSX keeps the prior
+    // token stream. `jsx_text_mode` is threaded as a COMPTIME parameter so the entire
+    // text-tracking path compiles away for non-JSX input — the hot lexer used by the
+    // overwhelming majority of files is byte-for-byte identical to before.
+    if (language.isJsx() and !language.isTs()) {
+        return tokenizeScalarImpl(true, alloc, src, language, opts);
+    }
+    return tokenizeScalarImpl(false, alloc, src, language, opts);
+}
+
+fn tokenizeScalarImpl(
+    comptime jsx_text_mode: bool,
+    alloc: std.mem.Allocator,
+    src: []const u8,
+    language: Language,
+    opts: Lex.TokenizeOptions,
 ) !TokenList {
     var toks: TokenList = .empty;
     try toks.ensureTotalCapacity(alloc, @max(src.len / 2 + 16, 64));
@@ -559,6 +599,22 @@ pub fn tokenizeScalarWithOptions(
     // classify a string as a JSX attribute value vs. an ordinary string.
     var jsx_tag_depth: u32 = 0;
     var jsx_brace_nest: u32 = 0;
+    // JSX element-body tracking, to emit one `jsx_text` token per text child (#61).
+    // The lexer is context-free, so it tracks JSX structure here: a stack of frames,
+    // each an element BODY (text context) or an expression container EXPR (JS
+    // context). `<tag>` pushes BODY, `</tag>` pops it; `{` inside a body pushes EXPR
+    // and its matching `}` pops it (the frame's low bits count nested object/block
+    // braces so the right `}` closes the container). `in_text` caches "top frame is a
+    // BODY and we are outside any tag header" — the single bit the hot loop tests.
+    //
+    // The whole struct is comptime-empty when jsx_text_mode is false, so the non-JSX
+    // (and TSX) instantiation of this worker carries none of it — no frame array on
+    // the stack, no structural-switch code — and is byte-for-byte identical to before.
+    var jx: JsxState(jsx_text_mode) = .{};
+    if (jsx_text_mode) jx.frame_ptr = &jx.frame_inline;
+    defer if (jsx_text_mode) {
+        if (jx.frame_heap) |h| alloc.free(h);
+    };
     // Annex B HTML comments (`<!--` / `-->`) are enabled only in non-module
     // scripts with annex_b set.
     const annex_b = opts.annex_b;
@@ -600,7 +656,23 @@ pub fn tokenizeScalarWithOptions(
         // one indirect branch instead of the former 11-deep if/else chain.
         // No-token cases (whitespace, comments, BOM/LS/PS) `continue`;
         // token-producing cases set `tag`/`i` and fall to the shared emit tail.
-        if (c == ' ' or c == '\t' or c == 0x0B or c == 0x0C) {
+        if (jsx_text_mode and jx.in_text and c != '<' and c != '{' and c != '>' and c != '}') {
+            // JSX text child (#61): in element-body context, everything up to the
+            // next `<` (tag) or `{` (expression container) is one literal jsx_text
+            // token, surrounding whitespace included. JSXText excludes `> }` too, so
+            // a bare `>`/`}` ends the run and is lexed as its own token — which the
+            // parser rejects, matching the reference. `jx.in_text` is only ever set
+            // when jsx_text_mode is on, so this branch is dead code off the JSX path.
+            @branchHint(.unlikely);
+            var j = i + 1;
+            var nl = c == '\n' or c == '\r';
+            while (j < n and src[j] != '<' and src[j] != '{' and src[j] != '>' and src[j] != '}') : (j += 1) {
+                if (src[j] == '\n' or src[j] == '\r') nl = true;
+            }
+            jx.text_nl = nl;
+            tag = .jsx_text;
+            i = j;
+        } else if (c == ' ' or c == '\t' or c == 0x0B or c == 0x0C) {
             i += 1;
             continue;
         } else if (c == '\n' or c == '\r') {
@@ -920,35 +992,138 @@ pub fn tokenizeScalarWithOptions(
         p_esc[t_len] = has_esc;
         t_len += 1;
 
-        // Maintain JSX opening-tag depth. `<` opens a JSX element when in
-        // expression/child position (regexAllowed) and followed by a tag-name
-        // start or `>` (fragment); `{`/`}` nest inside the tag header; `>`
-        // closes the header.
+        // Maintain JSX structure state. `<` opens an opening tag (in a body, or
+        // where a JSX expression may start) or a closing tag (`</`); `{`/`}` nest
+        // inside a tag header (attribute expression) or, in a body, open/close an
+        // expression container; `>` closes a header — entering the element body
+        // (opening tag) or leaving it (closing tag). See the var block above.
         if (is_jsx) {
-            switch (tag) {
-                .less_than => {
-                    if (Lex.regexAllowed(prev)) {
-                        const nb: u8 = if (i < n) src[i] else 0;
-                        const opens = nb == '>' or nb == '_' or nb == '$' or
-                            (nb >= 'a' and nb <= 'z') or (nb >= 'A' and nb <= 'Z') or nb >= 0x80;
-                        if (opens) jsx_tag_depth += 1;
-                    }
-                },
-                .l_brace => if (jsx_tag_depth > 0) {
-                    jsx_brace_nest += 1;
-                },
-                .r_brace => if (jsx_brace_nest > 0) {
-                    jsx_brace_nest -= 1;
-                },
-                .greater_than => if (jsx_tag_depth > 0 and jsx_brace_nest == 0) {
-                    jsx_tag_depth -= 1;
-                },
-                else => {},
+            if (jsx_text_mode) {
+                // Full JSX structure tracking (element bodies + expression containers)
+                // so body text becomes jsx_text tokens (#61). Compiled only into the
+                // plain-JSX instantiation.
+                switch (tag) {
+                    .less_than => {
+                        // In a body a `<` is always a tag (text can't contain a bare
+                        // `<`); elsewhere fall back to the expression-position heuristic.
+                        if (jx.in_text or Lex.regexAllowed(prev)) {
+                            const nb: u8 = if (i < n) src[i] else 0;
+                            if (nb == '/') {
+                                // Closing tag `</…>` — only meaningful inside a body.
+                                if (jx.sp > 0 and (jx.frame_ptr[jx.sp - 1] & JSX_EXPR_BIT) == 0) {
+                                    jx.closing = true;
+                                    jsx_tag_depth += 1; // header until the matching `>`
+                                }
+                            } else {
+                                const opens = nb == '>' or nb == '_' or nb == '$' or
+                                    (nb >= 'a' and nb <= 'z') or (nb >= 'A' and nb <= 'Z') or nb >= 0x80;
+                                if (opens) jsx_tag_depth += 1;
+                            }
+                        }
+                    },
+                    .l_brace => {
+                        if (jsx_tag_depth > 0) {
+                            jsx_brace_nest += 1; // brace inside a tag header
+                        } else if (jx.sp > 0) {
+                            const top = jx.frame_ptr[jx.sp - 1];
+                            if (top & JSX_EXPR_BIT == 0) {
+                                // `{` in a body opens an expression container.
+                                if (jx.sp == jx.frame_cap) {
+                                    const new_cap = jx.frame_cap * 2;
+                                    const grown = try alloc.alloc(u32, new_cap);
+                                    @memcpy(grown[0..jx.frame_cap], jx.frame_ptr[0..jx.frame_cap]);
+                                    if (jx.frame_heap) |h| alloc.free(h);
+                                    jx.frame_heap = grown;
+                                    jx.frame_ptr = grown.ptr;
+                                    jx.frame_cap = new_cap;
+                                }
+                                jx.frame_ptr[jx.sp] = JSX_EXPR_BIT;
+                                jx.sp += 1;
+                            } else {
+                                jx.frame_ptr[jx.sp - 1] = top + 1; // nested object/block brace
+                            }
+                        }
+                    },
+                    .r_brace => {
+                        if (jsx_brace_nest > 0) {
+                            jsx_brace_nest -= 1;
+                        } else if (jx.sp > 0) {
+                            const top = jx.frame_ptr[jx.sp - 1];
+                            if (top & JSX_EXPR_BIT != 0) {
+                                if (top & ~JSX_EXPR_BIT > 0) {
+                                    jx.frame_ptr[jx.sp - 1] = top - 1; // close a nested brace
+                                } else {
+                                    jx.sp -= 1; // close the expression container
+                                }
+                            }
+                        }
+                    },
+                    .greater_than => {
+                        if (jx.closing) {
+                            // `>` of a closing tag `</…>` — leave the element body.
+                            jx.closing = false;
+                            jsx_tag_depth -= 1;
+                            if (jx.sp > 0 and (jx.frame_ptr[jx.sp - 1] & JSX_EXPR_BIT) == 0) jx.sp -= 1;
+                        } else if (jsx_tag_depth > 0 and jsx_brace_nest == 0) {
+                            jsx_tag_depth -= 1;
+                            // Opening-tag header closed: enter the body unless self-closing `/>`.
+                            if (prev != .slash) {
+                                if (jx.sp == jx.frame_cap) {
+                                    const new_cap = jx.frame_cap * 2;
+                                    const grown = try alloc.alloc(u32, new_cap);
+                                    @memcpy(grown[0..jx.frame_cap], jx.frame_ptr[0..jx.frame_cap]);
+                                    if (jx.frame_heap) |h| alloc.free(h);
+                                    jx.frame_heap = grown;
+                                    jx.frame_ptr = grown.ptr;
+                                    jx.frame_cap = new_cap;
+                                }
+                                jx.frame_ptr[jx.sp] = 0; // BODY frame
+                                jx.sp += 1;
+                            }
+                        }
+                    },
+                    else => {},
+                }
+                // Recompute the hot-loop text-context bit: top frame is a body and we
+                // are outside any tag header / attribute brace.
+                jx.in_text = jx.sp > 0 and (jx.frame_ptr[jx.sp - 1] & JSX_EXPR_BIT) == 0 and
+                    jsx_tag_depth == 0 and jsx_brace_nest == 0;
+            } else {
+                // TSX / non-text JSX: only the opening-tag-header depth and attribute
+                // brace nesting needed to classify attribute strings (identical to the
+                // pre-#61 lexer). No element-body / text tracking.
+                switch (tag) {
+                    .less_than => {
+                        if (Lex.regexAllowed(prev)) {
+                            const nb: u8 = if (i < n) src[i] else 0;
+                            const opens = nb == '>' or nb == '_' or nb == '$' or
+                                (nb >= 'a' and nb <= 'z') or (nb >= 'A' and nb <= 'Z') or nb >= 0x80;
+                            if (opens) jsx_tag_depth += 1;
+                        }
+                    },
+                    .l_brace => if (jsx_tag_depth > 0) {
+                        jsx_brace_nest += 1;
+                    },
+                    .r_brace => if (jsx_brace_nest > 0) {
+                        jsx_brace_nest -= 1;
+                    },
+                    .greater_than => if (jsx_tag_depth > 0 and jsx_brace_nest == 0) {
+                        jsx_tag_depth -= 1;
+                    },
+                    else => {},
+                }
             }
         }
 
         prev_kind = if (isPropertyAccess(prev) and tag.isKeyword()) .identifier else tag;
-        saw_nl = false;
+        // A jsx_text token may span line terminators; carry that to the next token's
+        // has_newline_before. Comptime-gated so non-JSX keeps the plain `saw_nl = false`.
+        if (jsx_text_mode) {
+            saw_nl = jx.text_nl;
+            jx.text_nl = false;
+        } else {
+            saw_nl = false;
+        }
         at_line_start = false;
     }