From c1e7bf299a150a60f5fee9cb774fbadc9e2f6b8f Mon Sep 17 00:00:00 2001 From: Eric San Date: Thu, 25 Jun 2026 23:10:17 +0800 Subject: [PATCH 1/3] feat(lexer): emit one jsx_text token per JSX text child in .jsx (#61) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JSX child text was tokenized as ordinary JS tokens (an identifier per word) and the surrounding whitespace was dropped as trivia, so the token stream diverged from espree/@typescript-eslint (which emit one JSXText token spanning the whole text node). The `jsx_text` tag and the `jsx_text_node` AST node already existed with the correct range — only the token stream was wrong. The lexer is context-free, so it now tracks JSX structure itself: a stack of frames, each an element BODY (text context) or an expression container EXPR. `` pushes a BODY, `` pops it; `{` in a body pushes EXPR and the matching `}` pops it (the frame's low bits count nested object/block braces so the right `}` closes the container); self-closing `
` and fragments `<>` are handled. In body text context the scanner emits everything up to the next `< { > }` as one jsx_text token, whitespace included; a bare `>`/`}` ends the run and is lexed as its own token, which the parser rejects (matching the reference). Scoped to plain JSX (.jsx): in TSX a `` is ambiguous (JSX element vs generic type args vs generic arrow) and only the parser's speculative parse can disambiguate — committing in the lexer regressed 14 tsx/ts conformance cases, so element-body tracking is gated off for TS. TSX keeps its prior token stream (tracked as a follow-up). `jsx_text_mode` is threaded as a COMPTIME parameter so the whole text-tracking path compiles away for non-JSX input — the hot lexer for the vast majority of files is byte-for-byte identical to before (measured: no perf delta on a 274 KB non-JSX file). Validated: token streams match the reference exactly (the issue repro yields JSXText[5,18] + [37,38]); full suite green incl. test262 3966/3966 · 1389/1389; babel 1928/1928 · 1548/1548; TS conformance 17910/17913 · 1210/1223 — all at baseline. --- src/scalar_lexer.zig | 162 ++++++++++++++++++++++++++++++++++++++----- tests/lexer_test.zig | 54 +++++++++++++++ 2 files changed, 200 insertions(+), 16 deletions(-) diff --git a/src/scalar_lexer.zig b/src/scalar_lexer.zig index a1a2264..a791ca6 100644 --- a/src/scalar_lexer.zig +++ b/src/scalar_lexer.zig @@ -533,6 +533,26 @@ pub fn tokenizeScalarWithOptions( src: []const u8, language: Language, opts: Lex.TokenizeOptions, +) !TokenList { + // JSX text tokenization (#61) needs to know when a `` opens an element + // body. In TSX a `` is ambiguous (JSX element vs generic type arguments vs + // generic arrow), and only the parser's speculative parse can disambiguate, so + // element-body tracking is restricted to plain JSX (no TS); TSX keeps the prior + // token stream. `jsx_text_mode` is threaded as a COMPTIME parameter so the entire + // text-tracking path compiles away for non-JSX input — the hot lexer used by the + // overwhelming majority of files is byte-for-byte identical to before. + if (language.isJsx() and !language.isTs()) { + return tokenizeScalarImpl(true, alloc, src, language, opts); + } + return tokenizeScalarImpl(false, alloc, src, language, opts); +} + +fn tokenizeScalarImpl( + comptime jsx_text_mode: bool, + alloc: std.mem.Allocator, + src: []const u8, + language: Language, + opts: Lex.TokenizeOptions, ) !TokenList { var toks: TokenList = .empty; try toks.ensureTotalCapacity(alloc, @max(src.len / 2 + 16, 64)); @@ -559,6 +579,24 @@ pub fn tokenizeScalarWithOptions( // classify a string as a JSX attribute value vs. an ordinary string. var jsx_tag_depth: u32 = 0; var jsx_brace_nest: u32 = 0; + // JSX element-body tracking, to emit one `jsx_text` token per text child (#61). + // The lexer is context-free, so it tracks JSX structure here: a stack of frames, + // each an element BODY (text context) or an expression container EXPR (JS + // context). `` pushes BODY, `` pops it; `{` inside a body pushes EXPR + // and its matching `}` pops it (the frame's low bits count nested object/block + // braces so the right `}` closes the container). `jsx_in_text` caches "top frame + // is a BODY and we are outside any tag header" — the single bit the hot loop + // tests. All of this is gated on is_jsx, so non-JSX lexing is unchanged. + const JSX_EXPR_BIT: u32 = 0x8000_0000; + var jsx_in_text = false; + var jsx_closing = false; // the `<` just seen begins a closing tag ``) are enabled only in non-module // scripts with annex_b set. const annex_b = opts.annex_b; @@ -600,7 +638,23 @@ pub fn tokenizeScalarWithOptions( // one indirect branch instead of the former 11-deep if/else chain. // No-token cases (whitespace, comments, BOM/LS/PS) `continue`; // token-producing cases set `tag`/`i` and fall to the shared emit tail. - if (c == ' ' or c == '\t' or c == 0x0B or c == 0x0C) { + if (jsx_text_mode and jsx_in_text and c != '<' and c != '{' and c != '>' and c != '}') { + // JSX text child (#61): in element-body context, everything up to the + // next `<` (tag) or `{` (expression container) is one literal jsx_text + // token, surrounding whitespace included. JSXText excludes `> }` too, so + // a bare `>`/`}` ends the run and is lexed as its own token — which the + // parser rejects, matching the reference. `jsx_in_text` is false for all + // non-JSX input, so this branch never fires off the JSX path. + @branchHint(.unlikely); + var j = i + 1; + var nl = c == '\n' or c == '\r'; + while (j < n and src[j] != '<' and src[j] != '{' and src[j] != '>' and src[j] != '}') : (j += 1) { + if (src[j] == '\n' or src[j] == '\r') nl = true; + } + jsx_text_nl = nl; + tag = .jsx_text; + i = j; + } else if (c == ' ' or c == '\t' or c == 0x0B or c == 0x0C) { i += 1; continue; } else if (c == '\n' or c == '\r') { @@ -920,35 +974,111 @@ pub fn tokenizeScalarWithOptions( p_esc[t_len] = has_esc; t_len += 1; - // Maintain JSX opening-tag depth. `<` opens a JSX element when in - // expression/child position (regexAllowed) and followed by a tag-name - // start or `>` (fragment); `{`/`}` nest inside the tag header; `>` - // closes the header. + // Maintain JSX structure state. `<` opens an opening tag (in a body, or + // where a JSX expression may start) or a closing tag (`` closes a header — entering the element body + // (opening tag) or leaving it (closing tag). See the var block above. if (is_jsx) { switch (tag) { .less_than => { - if (Lex.regexAllowed(prev)) { + // In a body a `<` is always a tag (text can't contain a bare `<`); + // elsewhere fall back to the expression-position heuristic. + if (jsx_in_text or Lex.regexAllowed(prev)) { const nb: u8 = if (i < n) src[i] else 0; - const opens = nb == '>' or nb == '_' or nb == '$' or - (nb >= 'a' and nb <= 'z') or (nb >= 'A' and nb <= 'Z') or nb >= 0x80; - if (opens) jsx_tag_depth += 1; + if (nb == '/') { + // Closing tag `` — only meaningful directly in a body. + if (jsx_sp > 0 and (jsx_frame_ptr[jsx_sp - 1] & JSX_EXPR_BIT) == 0) { + jsx_closing = true; + jsx_tag_depth += 1; // header until the matching `>` + } + } else { + const opens = nb == '>' or nb == '_' or nb == '$' or + (nb >= 'a' and nb <= 'z') or (nb >= 'A' and nb <= 'Z') or nb >= 0x80; + if (opens) jsx_tag_depth += 1; + } } }, - .l_brace => if (jsx_tag_depth > 0) { - jsx_brace_nest += 1; + .l_brace => { + if (jsx_tag_depth > 0) { + jsx_brace_nest += 1; // brace inside a tag header + } else if (jsx_sp > 0) { + const top = jsx_frame_ptr[jsx_sp - 1]; + if (top & JSX_EXPR_BIT == 0) { + // `{` in a body opens an expression container. + if (jsx_sp == jsx_frame_cap) { + const new_cap = jsx_frame_cap * 2; + const grown = try alloc.alloc(u32, new_cap); + @memcpy(grown[0..jsx_frame_cap], jsx_frame_ptr[0..jsx_frame_cap]); + if (jsx_frame_heap) |h| alloc.free(h); + jsx_frame_heap = grown; + jsx_frame_ptr = grown.ptr; + jsx_frame_cap = new_cap; + } + jsx_frame_ptr[jsx_sp] = JSX_EXPR_BIT; + jsx_sp += 1; + } else { + jsx_frame_ptr[jsx_sp - 1] = top + 1; // nested object/block brace + } + } }, - .r_brace => if (jsx_brace_nest > 0) { - jsx_brace_nest -= 1; + .r_brace => { + if (jsx_brace_nest > 0) { + jsx_brace_nest -= 1; + } else if (jsx_sp > 0) { + const top = jsx_frame_ptr[jsx_sp - 1]; + if (top & JSX_EXPR_BIT != 0) { + if (top & ~JSX_EXPR_BIT > 0) { + jsx_frame_ptr[jsx_sp - 1] = top - 1; // close a nested brace + } else { + jsx_sp -= 1; // close the expression container + } + } + } }, - .greater_than => if (jsx_tag_depth > 0 and jsx_brace_nest == 0) { - jsx_tag_depth -= 1; + .greater_than => { + if (jsx_closing) { + // `>` of a closing tag `` — leave the element body. + jsx_closing = false; + jsx_tag_depth -= 1; + if (jsx_sp > 0 and (jsx_frame_ptr[jsx_sp - 1] & JSX_EXPR_BIT) == 0) jsx_sp -= 1; + } else if (jsx_tag_depth > 0 and jsx_brace_nest == 0) { + jsx_tag_depth -= 1; + // Opening-tag header closed: enter the body unless self-closing `/>`. + // Gated to plain JSX — see jsx_text_mode. (No body pushed in TSX, so + // jsx_sp stays 0 and all body/expr/text tracking stays inert.) + if (prev != .slash and jsx_text_mode) { + if (jsx_sp == jsx_frame_cap) { + const new_cap = jsx_frame_cap * 2; + const grown = try alloc.alloc(u32, new_cap); + @memcpy(grown[0..jsx_frame_cap], jsx_frame_ptr[0..jsx_frame_cap]); + if (jsx_frame_heap) |h| alloc.free(h); + jsx_frame_heap = grown; + jsx_frame_ptr = grown.ptr; + jsx_frame_cap = new_cap; + } + jsx_frame_ptr[jsx_sp] = 0; // BODY frame + jsx_sp += 1; + } + } }, else => {}, } + // Recompute the hot-loop text-context bit: top frame is a body and we + // are outside any tag header / attribute brace. + jsx_in_text = jsx_sp > 0 and (jsx_frame_ptr[jsx_sp - 1] & JSX_EXPR_BIT) == 0 and + jsx_tag_depth == 0 and jsx_brace_nest == 0; } prev_kind = if (isPropertyAccess(prev) and tag.isKeyword()) .identifier else tag; - saw_nl = false; + // A jsx_text token may span line terminators; carry that to the next token's + // has_newline_before. Comptime-gated so non-JSX keeps the plain `saw_nl = false`. + if (jsx_text_mode) { + saw_nl = jsx_text_nl; + jsx_text_nl = false; + } else { + saw_nl = false; + } at_line_start = false; } diff --git a/tests/lexer_test.zig b/tests/lexer_test.zig index db9500a..decd338 100644 --- a/tests/lexer_test.zig +++ b/tests/lexer_test.zig @@ -357,3 +357,57 @@ test "only whitespace" { test "only comments" { try expectTokens("// comment\n/* block */", &.{}); } + +// ── JSX text tokens (#61) ──────────────────────────────── + +/// Collect the [start, end) byte ranges of every jsx_text token, in order. +fn expectJsxTextLang(src: []const u8, lang: Token.Language, expected: []const [2]u32) !void { + var result = try Lexer.tokenizeWithLanguage(testing.allocator, src, lang); + defer result.deinit(testing.allocator); + const tags = result.tokens.items(.tag); + const starts = result.tokens.items(.start); + const lens = result.tokens.items(.len); + var got: [16][2]u32 = undefined; + var k: usize = 0; + var i: usize = 0; + while (i < result.tokens.len) : (i += 1) { + if (tags[i] == .jsx_text) { + got[k] = .{ starts[i], starts[i] + lens[i] }; + k += 1; + } + } + try testing.expectEqual(expected.len, k); + for (expected, got[0..k]) |e, g| { + try testing.expectEqual(e[0], g[0]); + try testing.expectEqual(e[1], g[1]); + } +} + +fn expectJsxText(src: []const u8, expected: []const [2]u32) !void { + try expectJsxTextLang(src, .jsx, expected); +} + +test "JSX text child is one jsx_text token spanning whitespace (#61)" { + // The issue's repro: text before/after the expression container are each ONE + // jsx_text token including the leading "\n " — matching espree/@typescript-eslint. + try expectJsxText("
\n unrelated{\n foo\n }\n
", &.{ .{ 5, 18 }, .{ 37, 38 } }); + // Multi-word text is one token, not several identifiers. + try expectJsxText("
hello world
", &.{.{ 5, 16 }}); + // Nested elements: only the innermost text is jsx_text; adjacent tags have none. + try expectJsxText("x", &.{.{ 6, 7 }}); + // Text inside a nested element within an expression container. + try expectJsxText("
{cond && hi}
", &.{.{ 20, 22 }}); + // Fragment body. + try expectJsxText("<>frag", &.{.{ 2, 6 }}); + // A self-closing element opens no body — the surrounding gaps are not jsx_text. + try expectJsxText("

", &.{}); +} + +test "JSX text tokens are gated to plain JSX, not TSX / non-JSX (#61)" { + // Plain JS: `<` / `>` are operators, never JSX tags. + try expectJsxTextLang("a < b > c", .js, &.{}); + // TSX keeps the prior token stream (the JSX-vs-generic ambiguity needs the + // parser); a generic arrow must NOT be misread as a JSX element opening a body. + try expectJsxTextLang("const f = () => 1;", .tsx, &.{}); + try expectJsxTextLang("
hi
", .tsx, &.{}); +} From 120cde8974ec570cebc1077b4bde55d1f63b2eb8 Mon Sep 17 00:00:00 2001 From: Eric San Date: Thu, 25 Jun 2026 23:27:21 +0800 Subject: [PATCH 2/3] =?UTF-8?q?test(lexer):=20#61=20review=20=E2=80=94=20c?= =?UTF-8?q?over=20frame-spill,=20newline=20carry,=20nested-brace=20paths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review: add the three untested code paths the change introduced — the heap frame-stack spill (>64-deep nesting, the only allocating path), the has_newline_before carry after a multi-line jsx_text token, and the expression-container brace counting (object literal whose inner `}` must not close the container early). Also cover parent-body trailing text and whitespace-only text, and bump the helper's range buffer 16 → 64 so a many-text-child input can't index out of bounds. (Reviews otherwise clean: correctness verified under a 2M-iteration fuzz; conformance byte-identical to main; comptime gating proven by codegen size with no measurable non-JSX perf delta.) --- tests/lexer_test.zig | 45 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/lexer_test.zig b/tests/lexer_test.zig index decd338..7b807cb 100644 --- a/tests/lexer_test.zig +++ b/tests/lexer_test.zig @@ -367,7 +367,7 @@ fn expectJsxTextLang(src: []const u8, lang: Token.Language, expected: []const [2 const tags = result.tokens.items(.tag); const starts = result.tokens.items(.start); const lens = result.tokens.items(.len); - var got: [16][2]u32 = undefined; + var got: [64][2]u32 = undefined; var k: usize = 0; var i: usize = 0; while (i < result.tokens.len) : (i += 1) { @@ -411,3 +411,46 @@ test "JSX text tokens are gated to plain JSX, not TSX / non-JSX (#61)" { try expectJsxTextLang("const f = () => 1;", .tsx, &.{}); try expectJsxTextLang("
hi
", .tsx, &.{}); } + +test "JSX text: nested-brace, parent-body and whitespace-only paths (#61)" { + // Object literal inside an expression container: the inner `}` must NOT close + // the container early (exercises the frame brace-count low bits) → no text. + try expectJsxText("
{ {a:1} }
", &.{}); + // Trailing text in the PARENT body after a child element closes. + try expectJsxText("xy", &.{ .{ 6, 7 }, .{ 11, 12 } }); + // Whitespace-only text is still one jsx_text token. + try expectJsxText("
", &.{.{ 5, 8 }}); +} + +test "JSX text: a line terminator in text carries to the next token (#61)" { + var result = try Lexer.tokenizeWithLanguage(testing.allocator, "
l1\nl2
", .jsx); + defer result.deinit(testing.allocator); + const tags = result.tokens.items(.tag); + const nl = result.tokens.items(.has_newline_before); + var i: usize = 0; + while (tags[i] != .jsx_text) : (i += 1) {} + try testing.expect(!nl[i]); // the text token itself: no newline before it + try testing.expect(nl[i + 1]); // the following `<` sees the in-text newline +} + +test "JSX text: deep nesting spills the inline frame stack (#61)" { + // >64 BODY frames forces the heap-grow path (jsx_frame_heap) — the only + // allocating path in the change; the inner text must still be one token. + const alloc = testing.allocator; + var src: std.ArrayListUnmanaged(u8) = .{ .items = &.{}, .capacity = 0 }; + defer src.deinit(alloc); + var i: usize = 0; + while (i < 70) : (i += 1) try src.appendSlice(alloc, ""); + try src.append(alloc, 'x'); + i = 0; + while (i < 70) : (i += 1) try src.appendSlice(alloc, ""); + + var result = try Lexer.tokenizeWithLanguage(alloc, src.items, .jsx); + defer result.deinit(alloc); + const tags = result.tokens.items(.tag); + var k: usize = 0; + for (0..result.tokens.len) |idx| { + if (tags[idx] == .jsx_text) k += 1; + } + try testing.expectEqual(@as(usize, 1), k); +} From 2b1ab10a8084c50393e1dcd45f12debd5bf66d05 Mon Sep 17 00:00:00 2001 From: Eric San Date: Fri, 26 Jun 2026 09:27:05 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix(lexer):=20#61=20fan-out=20review=20?= =?UTF-8?q?=E2=80=94=20zero-cost=20gating,=20tokenText=20crash,=20test=20g?= =?UTF-8?q?aps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial fan-out review (8 dimensions, each finding independently reproduced) surfaced one latent crash, one real perf regression, and test gaps. Fixes: - perf: the prior gating left the [64]u32 frame array + the structural-switch code in the `false` instantiation (shared by non-JSX and TSX), costing a measured ~2% on the non-JSX hot lexer (interleaved median 3.68 vs 3.60s). Move the JSX-text state into a comptime-conditional `JsxState(enabled)` struct — empty (zero-size) when off — and comptime-split the structural switch into the full tracker (plain JSX) vs the original tag-header/attr-brace tracker (TSX). The non-JSX worker is now byte-identical to main: re-benchmarked at parity (median 3.49 vs 3.53s), frame array gone. - crash: Parser.tokenText null-deref'd on the new jsx_text token (ordinal 130 fell outside its variable-lexeme fast-path range, and lexeme(.jsx_text) is null). Latent today (no caller hits it — parseJsxChildren uses tok span fields) but a landmine; add an explicit jsx_text case and fix the stale comment that wrongly claimed jsx_text sat in the trailing range. - tests: the self-closing assertion didn't actually guard `prev != .slash` (bug-injection still passed) — replace with `
tail` -> none. Add the two untested paths: the EXPR-container heap-grow site and bare `>`/`}` text-splitting. Accepted (noted, not fixed): a bare `}` after only whitespace (`
}
`, invalid input) loses one diagnostic main happened to emit — no valid program or conformance baseline is affected, and it merely makes the parser's already-lenient bare-`}` handling uniform. Validated: full suite green incl. test262 3966/3966 · 1389/1389; babel 1928/1928 · 1548/1548; TS 17910/17913 · 1210/1223; jsx_text token streams unchanged. --- src/parser.zig | 11 +- src/scalar_lexer.zig | 241 +++++++++++++++++++++++++------------------ tests/lexer_test.zig | 40 ++++++- 3 files changed, 188 insertions(+), 104 deletions(-) diff --git a/src/parser.zig b/src/parser.zig index 2eb0eee..3713ef2 100644 --- a/src/parser.zig +++ b/src/parser.zig @@ -904,11 +904,14 @@ pub const Parser = struct { /// @returns borrowed_from(self) pub fn tokenText(self: *const Parser, index: TokenIndex) []const u8 { const tag = self.tags_ptr[index]; - // Variable-lexeme tokens (identifiers, literals, escaped_keyword, etc.) are the first - // 9 enum variants (0..identifier) plus the last 4 (escaped_keyword..jsx_text). - // A range check is cheaper than the 80-arm lexeme() switch for these common cases. + // Variable-lexeme tokens carry their own source text. They are the first 9 + // enum variants (0..identifier) and the tail group escaped_keyword/at_sign/ + // hashbang — a range check is cheaper than the 80-arm lexeme() switch. jsx_text + // is also variable-lexeme but sits before eof in the enum (outside the tail + // range), so it needs an explicit case; lexeme(.jsx_text) is null, so without + // this it would null-deref. (eof/invalid keep their fixed ``/``.) const ti = @intFromEnum(tag); - if (ti <= @intFromEnum(TokenTag.identifier) or ti >= @intFromEnum(TokenTag.escaped_keyword)) { + if (ti <= @intFromEnum(TokenTag.identifier) or ti >= @intFromEnum(TokenTag.escaped_keyword) or tag == .jsx_text) { const start = self.tok_starts_ptr[index]; const len = self.tok_lens_ptr[index]; return self.source[start .. start + len]; diff --git a/src/scalar_lexer.zig b/src/scalar_lexer.zig index a791ca6..76cf545 100644 --- a/src/scalar_lexer.zig +++ b/src/scalar_lexer.zig @@ -486,6 +486,26 @@ pub const Token = struct { has_unicode_escape: bool = false, }; +/// Top bit of a JSX frame marks an expression container (vs an element body); +/// the low bits count its nested object/block braces (#61). +const JSX_EXPR_BIT: u32 = 0x8000_0000; + +/// Per-call JSX element-body tracking state (#61). Comptime-conditional: returns a +/// zero-size struct when text tokenization is off, so the non-JSX / TSX instantiation +/// of the lexer carries no frame array and no tracking fields at all. +fn JsxState(comptime enabled: bool) type { + if (!enabled) return struct {}; + return struct { + in_text: bool = false, + closing: bool = false, // the `<` just seen begins a closing tag `` pushes BODY, `` pops it; `{` inside a body pushes EXPR // and its matching `}` pops it (the frame's low bits count nested object/block - // braces so the right `}` closes the container). `jsx_in_text` caches "top frame - // is a BODY and we are outside any tag header" — the single bit the hot loop - // tests. All of this is gated on is_jsx, so non-JSX lexing is unchanged. - const JSX_EXPR_BIT: u32 = 0x8000_0000; - var jsx_in_text = false; - var jsx_closing = false; // the `<` just seen begins a closing tag ``) are enabled only in non-module // scripts with annex_b set. const annex_b = opts.annex_b; @@ -638,20 +656,20 @@ fn tokenizeScalarImpl( // one indirect branch instead of the former 11-deep if/else chain. // No-token cases (whitespace, comments, BOM/LS/PS) `continue`; // token-producing cases set `tag`/`i` and fall to the shared emit tail. - if (jsx_text_mode and jsx_in_text and c != '<' and c != '{' and c != '>' and c != '}') { + if (jsx_text_mode and jx.in_text and c != '<' and c != '{' and c != '>' and c != '}') { // JSX text child (#61): in element-body context, everything up to the // next `<` (tag) or `{` (expression container) is one literal jsx_text // token, surrounding whitespace included. JSXText excludes `> }` too, so // a bare `>`/`}` ends the run and is lexed as its own token — which the - // parser rejects, matching the reference. `jsx_in_text` is false for all - // non-JSX input, so this branch never fires off the JSX path. + // parser rejects, matching the reference. `jx.in_text` is only ever set + // when jsx_text_mode is on, so this branch is dead code off the JSX path. @branchHint(.unlikely); var j = i + 1; var nl = c == '\n' or c == '\r'; while (j < n and src[j] != '<' and src[j] != '{' and src[j] != '>' and src[j] != '}') : (j += 1) { if (src[j] == '\n' or src[j] == '\r') nl = true; } - jsx_text_nl = nl; + jx.text_nl = nl; tag = .jsx_text; i = j; } else if (c == ' ' or c == '\t' or c == 0x0B or c == 0x0C) { @@ -980,102 +998,129 @@ fn tokenizeScalarImpl( // expression container; `>` closes a header — entering the element body // (opening tag) or leaving it (closing tag). See the var block above. if (is_jsx) { - switch (tag) { - .less_than => { - // In a body a `<` is always a tag (text can't contain a bare `<`); - // elsewhere fall back to the expression-position heuristic. - if (jsx_in_text or Lex.regexAllowed(prev)) { - const nb: u8 = if (i < n) src[i] else 0; - if (nb == '/') { - // Closing tag `` — only meaningful directly in a body. - if (jsx_sp > 0 and (jsx_frame_ptr[jsx_sp - 1] & JSX_EXPR_BIT) == 0) { - jsx_closing = true; - jsx_tag_depth += 1; // header until the matching `>` + if (jsx_text_mode) { + // Full JSX structure tracking (element bodies + expression containers) + // so body text becomes jsx_text tokens (#61). Compiled only into the + // plain-JSX instantiation. + switch (tag) { + .less_than => { + // In a body a `<` is always a tag (text can't contain a bare + // `<`); elsewhere fall back to the expression-position heuristic. + if (jx.in_text or Lex.regexAllowed(prev)) { + const nb: u8 = if (i < n) src[i] else 0; + if (nb == '/') { + // Closing tag `` — only meaningful inside a body. + if (jx.sp > 0 and (jx.frame_ptr[jx.sp - 1] & JSX_EXPR_BIT) == 0) { + jx.closing = true; + jsx_tag_depth += 1; // header until the matching `>` + } + } else { + const opens = nb == '>' or nb == '_' or nb == '$' or + (nb >= 'a' and nb <= 'z') or (nb >= 'A' and nb <= 'Z') or nb >= 0x80; + if (opens) jsx_tag_depth += 1; } - } else { - const opens = nb == '>' or nb == '_' or nb == '$' or - (nb >= 'a' and nb <= 'z') or (nb >= 'A' and nb <= 'Z') or nb >= 0x80; - if (opens) jsx_tag_depth += 1; } - } - }, - .l_brace => { - if (jsx_tag_depth > 0) { - jsx_brace_nest += 1; // brace inside a tag header - } else if (jsx_sp > 0) { - const top = jsx_frame_ptr[jsx_sp - 1]; - if (top & JSX_EXPR_BIT == 0) { - // `{` in a body opens an expression container. - if (jsx_sp == jsx_frame_cap) { - const new_cap = jsx_frame_cap * 2; - const grown = try alloc.alloc(u32, new_cap); - @memcpy(grown[0..jsx_frame_cap], jsx_frame_ptr[0..jsx_frame_cap]); - if (jsx_frame_heap) |h| alloc.free(h); - jsx_frame_heap = grown; - jsx_frame_ptr = grown.ptr; - jsx_frame_cap = new_cap; + }, + .l_brace => { + if (jsx_tag_depth > 0) { + jsx_brace_nest += 1; // brace inside a tag header + } else if (jx.sp > 0) { + const top = jx.frame_ptr[jx.sp - 1]; + if (top & JSX_EXPR_BIT == 0) { + // `{` in a body opens an expression container. + if (jx.sp == jx.frame_cap) { + const new_cap = jx.frame_cap * 2; + const grown = try alloc.alloc(u32, new_cap); + @memcpy(grown[0..jx.frame_cap], jx.frame_ptr[0..jx.frame_cap]); + if (jx.frame_heap) |h| alloc.free(h); + jx.frame_heap = grown; + jx.frame_ptr = grown.ptr; + jx.frame_cap = new_cap; + } + jx.frame_ptr[jx.sp] = JSX_EXPR_BIT; + jx.sp += 1; + } else { + jx.frame_ptr[jx.sp - 1] = top + 1; // nested object/block brace } - jsx_frame_ptr[jsx_sp] = JSX_EXPR_BIT; - jsx_sp += 1; - } else { - jsx_frame_ptr[jsx_sp - 1] = top + 1; // nested object/block brace } - } - }, - .r_brace => { - if (jsx_brace_nest > 0) { - jsx_brace_nest -= 1; - } else if (jsx_sp > 0) { - const top = jsx_frame_ptr[jsx_sp - 1]; - if (top & JSX_EXPR_BIT != 0) { - if (top & ~JSX_EXPR_BIT > 0) { - jsx_frame_ptr[jsx_sp - 1] = top - 1; // close a nested brace - } else { - jsx_sp -= 1; // close the expression container + }, + .r_brace => { + if (jsx_brace_nest > 0) { + jsx_brace_nest -= 1; + } else if (jx.sp > 0) { + const top = jx.frame_ptr[jx.sp - 1]; + if (top & JSX_EXPR_BIT != 0) { + if (top & ~JSX_EXPR_BIT > 0) { + jx.frame_ptr[jx.sp - 1] = top - 1; // close a nested brace + } else { + jx.sp -= 1; // close the expression container + } } } - } - }, - .greater_than => { - if (jsx_closing) { - // `>` of a closing tag `` — leave the element body. - jsx_closing = false; - jsx_tag_depth -= 1; - if (jsx_sp > 0 and (jsx_frame_ptr[jsx_sp - 1] & JSX_EXPR_BIT) == 0) jsx_sp -= 1; - } else if (jsx_tag_depth > 0 and jsx_brace_nest == 0) { - jsx_tag_depth -= 1; - // Opening-tag header closed: enter the body unless self-closing `/>`. - // Gated to plain JSX — see jsx_text_mode. (No body pushed in TSX, so - // jsx_sp stays 0 and all body/expr/text tracking stays inert.) - if (prev != .slash and jsx_text_mode) { - if (jsx_sp == jsx_frame_cap) { - const new_cap = jsx_frame_cap * 2; - const grown = try alloc.alloc(u32, new_cap); - @memcpy(grown[0..jsx_frame_cap], jsx_frame_ptr[0..jsx_frame_cap]); - if (jsx_frame_heap) |h| alloc.free(h); - jsx_frame_heap = grown; - jsx_frame_ptr = grown.ptr; - jsx_frame_cap = new_cap; + }, + .greater_than => { + if (jx.closing) { + // `>` of a closing tag `` — leave the element body. + jx.closing = false; + jsx_tag_depth -= 1; + if (jx.sp > 0 and (jx.frame_ptr[jx.sp - 1] & JSX_EXPR_BIT) == 0) jx.sp -= 1; + } else if (jsx_tag_depth > 0 and jsx_brace_nest == 0) { + jsx_tag_depth -= 1; + // Opening-tag header closed: enter the body unless self-closing `/>`. + if (prev != .slash) { + if (jx.sp == jx.frame_cap) { + const new_cap = jx.frame_cap * 2; + const grown = try alloc.alloc(u32, new_cap); + @memcpy(grown[0..jx.frame_cap], jx.frame_ptr[0..jx.frame_cap]); + if (jx.frame_heap) |h| alloc.free(h); + jx.frame_heap = grown; + jx.frame_ptr = grown.ptr; + jx.frame_cap = new_cap; + } + jx.frame_ptr[jx.sp] = 0; // BODY frame + jx.sp += 1; } - jsx_frame_ptr[jsx_sp] = 0; // BODY frame - jsx_sp += 1; } - } - }, - else => {}, + }, + else => {}, + } + // Recompute the hot-loop text-context bit: top frame is a body and we + // are outside any tag header / attribute brace. + jx.in_text = jx.sp > 0 and (jx.frame_ptr[jx.sp - 1] & JSX_EXPR_BIT) == 0 and + jsx_tag_depth == 0 and jsx_brace_nest == 0; + } else { + // TSX / non-text JSX: only the opening-tag-header depth and attribute + // brace nesting needed to classify attribute strings (identical to the + // pre-#61 lexer). No element-body / text tracking. + switch (tag) { + .less_than => { + if (Lex.regexAllowed(prev)) { + const nb: u8 = if (i < n) src[i] else 0; + const opens = nb == '>' or nb == '_' or nb == '$' or + (nb >= 'a' and nb <= 'z') or (nb >= 'A' and nb <= 'Z') or nb >= 0x80; + if (opens) jsx_tag_depth += 1; + } + }, + .l_brace => if (jsx_tag_depth > 0) { + jsx_brace_nest += 1; + }, + .r_brace => if (jsx_brace_nest > 0) { + jsx_brace_nest -= 1; + }, + .greater_than => if (jsx_tag_depth > 0 and jsx_brace_nest == 0) { + jsx_tag_depth -= 1; + }, + else => {}, + } } - // Recompute the hot-loop text-context bit: top frame is a body and we - // are outside any tag header / attribute brace. - jsx_in_text = jsx_sp > 0 and (jsx_frame_ptr[jsx_sp - 1] & JSX_EXPR_BIT) == 0 and - jsx_tag_depth == 0 and jsx_brace_nest == 0; } prev_kind = if (isPropertyAccess(prev) and tag.isKeyword()) .identifier else tag; // A jsx_text token may span line terminators; carry that to the next token's // has_newline_before. Comptime-gated so non-JSX keeps the plain `saw_nl = false`. if (jsx_text_mode) { - saw_nl = jsx_text_nl; - jsx_text_nl = false; + saw_nl = jx.text_nl; + jx.text_nl = false; } else { saw_nl = false; } diff --git a/tests/lexer_test.zig b/tests/lexer_test.zig index 7b807cb..b11e398 100644 --- a/tests/lexer_test.zig +++ b/tests/lexer_test.zig @@ -399,8 +399,13 @@ test "JSX text child is one jsx_text token spanning whitespace (#61)" { try expectJsxText("
{cond && hi}
", &.{.{ 20, 22 }}); // Fragment body. try expectJsxText("<>frag", &.{.{ 2, 6 }}); - // A self-closing element opens no body — the surrounding gaps are not jsx_text. - try expectJsxText("

", &.{}); + // A self-closing element opens NO body. At top level the trailing text is then + // plain JS, not jsx_text — this fails if `
` wrongly opens a body (genuinely + // exercises the `prev != .slash` guard; the old `

` assertion + // passed even with that guard removed, since there was no text to capture). + try expectJsxText("
tail", &.{}); + // Text after a self-closing child belongs to the ENCLOSING element's body. + try expectJsxText("
after", &.{.{ 8, 13 }}); } test "JSX text tokens are gated to plain JSX, not TSX / non-JSX (#61)" { @@ -454,3 +459,34 @@ test "JSX text: deep nesting spills the inline frame stack (#61)" { } try testing.expectEqual(@as(usize, 1), k); } + +test "JSX text: a bare > or } ends the text run as its own token (#61)" { + // JSXText excludes `>` and `}`; a bare one ends the run and is lexed separately + // (the parser then rejects the stray token). `
a>b
` → "a", `>`, "b". + try expectJsxText("
a>b
", &.{ .{ 5, 6 }, .{ 7, 8 } }); + // A leading `}` is its own r_brace token; the text run resumes after it. + try expectJsxText("
}x
", &.{.{ 6, 7 }}); +} + +test "JSX text: expression-container nesting spills the frame stack (#61)" { + // 64 BODY frames fill the inline stack, then a `{` pushes an EXPR frame at + // capacity — exercising the EXPR-container heap-grow site (the all-`` spill + // test only hits the BODY-frame grow). + const alloc = testing.allocator; + var src: std.ArrayListUnmanaged(u8) = .{ .items = &.{}, .capacity = 0 }; + defer src.deinit(alloc); + var i: usize = 0; + while (i < 64) : (i += 1) try src.appendSlice(alloc, ""); + try src.appendSlice(alloc, "{x}T"); // `{` pushes EXPR at sp==cap → grow; "T" is body text + i = 0; + while (i < 64) : (i += 1) try src.appendSlice(alloc, ""); + + var result = try Lexer.tokenizeWithLanguage(alloc, src.items, .jsx); + defer result.deinit(alloc); + const tags = result.tokens.items(.tag); + var k: usize = 0; + for (0..result.tokens.len) |idx| { + if (tags[idx] == .jsx_text) k += 1; + } + try testing.expectEqual(@as(usize, 1), k); // only "T" +}