diff --git a/.bca-baseline.toml b/.bca-baseline.toml index a3d54d44..2980efe2 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -189,19 +189,19 @@ value = 6.0 path = "big-code-analysis-ast/src/ast.rs" qualified = "build" metric = "halstead.effort" -value = 87299.1671815905 +value = 92648.17221763342 [[entry]] path = "big-code-analysis-ast/src/c_macro.rs" qualified = "replace" metric = "halstead.effort" -value = 82074.58162485641 +value = 86055.10448022334 [[entry]] path = "big-code-analysis-ast/src/c_macro.rs" qualified = "step_normal" metric = "halstead.effort" -value = 66515.67234293568 +value = 70211.8711610781 [[entry]] path = "big-code-analysis-ast/src/c_macro.rs" @@ -221,29 +221,35 @@ qualified = "step_raw_string" metric = "nargs" value = 5.0 +[[entry]] +path = "big-code-analysis-ast/src/getter.rs" +qualified = "Getter::is_braced_literal_slot" +metric = "halstead.effort" +value = 48009.70056636167 + [[entry]] path = "big-code-analysis-ast/src/getter/bash.rs" qualified = "BashCode::get_op_type" metric = "halstead.effort" -value = 57421.05047463073 +value = 57299.9090179332 [[entry]] path = "big-code-analysis-ast/src/getter/elixir.rs" qualified = "ElixirCode::get_op_type" metric = "halstead.effort" -value = 68425.65670070829 +value = 76840.62078430264 [[entry]] path = "big-code-analysis-ast/src/getter/irules.rs" qualified = "IrulesCode::get_op_type" metric = "halstead.effort" -value = 51602.8394982239 +value = 51477.285144213616 [[entry]] path = "big-code-analysis-ast/src/getter/perl.rs" qualified = "PerlCode::get_op_type" metric = "halstead.effort" -value = 106425.53731125958 +value = 131000.62269483795 [[entry]] path = "big-code-analysis-ast/src/getter/php.rs" @@ -255,25 +261,25 @@ value = 92711.94569253008 path = "big-code-analysis-ast/src/getter/python.rs" qualified = "PythonCode::get_op_type" metric = "halstead.effort" -value = 48346.66496041094 +value = 61900.01893104293 [[entry]] path = "big-code-analysis-ast/src/getter/ruby.rs" qualified = "RubyCode::get_op_type" metric = "halstead.effort" -value = 120805.48245794396 +value = 120642.89096069236 [[entry]] path = "big-code-analysis-ast/src/node.rs" qualified = "Node<'a>" metric = "nom" -value = 36.0 +value = 42.0 [[entry]] path = "big-code-analysis-ast/src/parser.rs" qualified = "Parser::filters" metric = "halstead.effort" -value = 85508.52631578948 +value = 87390.78947368421 [[entry]] path = "big-code-analysis-ast/src/preproc.rs" @@ -303,19 +309,19 @@ value = 7.0 path = "big-code-analysis-cli/src/baseline.rs" qualified = "Baseline::from_str" metric = "halstead.effort" -value = 58410.89968953587 +value = 61984.213363755734 [[entry]] path = "big-code-analysis-cli/src/baseline_diff.rs" qualified = "BaselineDiff::compute" metric = "halstead.effort" -value = 118801.67275912332 +value = 118241.28751025953 [[entry]] path = "big-code-analysis-cli/src/check_format.rs" qualified = "write_github_annotations" metric = "halstead.effort" -value = 47834.17129195417 +value = 48005.61993457766 [[entry]] path = "big-code-analysis-cli/src/commands.rs" @@ -327,13 +333,13 @@ value = 136001.1105471366 path = "big-code-analysis-cli/src/commands/analyze.rs" qualified = "run_command_metrics" metric = "halstead.effort" -value = 69608.24451083591 +value = 70105.44625734187 [[entry]] path = "big-code-analysis-cli/src/commands/analyze.rs" qualified = "write_aggregate" metric = "halstead.effort" -value = 54210.895759662584 +value = 54557.29125972433 [[entry]] path = "big-code-analysis-cli/src/commands/check.rs" @@ -357,7 +363,7 @@ value = 6.0 path = "big-code-analysis-cli/src/commands/check.rs" qualified = "run_check" metric = "halstead.effort" -value = 63636.29428662941 +value = 63799.464271979734 [[entry]] path = "big-code-analysis-cli/src/commands/check/effective_config.rs" @@ -405,13 +411,13 @@ value = 39.06404996924922 path = "big-code-analysis-cli/src/commands/report.rs" qualified = "run_command_report" metric = "halstead.effort" -value = 63112.954051595756 +value = 63991.55666020266 [[entry]] path = "big-code-analysis-cli/src/diff.rs" qualified = "materialize_tree" metric = "halstead.effort" -value = 60390.01093335322 +value = 60574.68986587111 [[entry]] path = "big-code-analysis-cli/src/diff.rs" @@ -557,6 +563,12 @@ qualified = "legacy_hint" metric = "halstead.effort" value = 59351.805921378844 +[[entry]] +path = "big-code-analysis-cli/src/manifest.rs" +qualified = "Manifest::merge_check" +metric = "halstead.effort" +value = 49443.19551903411 + [[entry]] path = "big-code-analysis-cli/src/markdown_report.rs" qualified = "" @@ -615,13 +627,13 @@ value = 7.0 path = "big-code-analysis-cli/src/threshold_suggestion.rs" qualified = "closest_names" metric = "halstead.effort" -value = 55785.628204736706 +value = 59984.93899693509 [[entry]] path = "big-code-analysis-cli/src/threshold_suggestion.rs" qualified = "edit_distance_with_cutoff" metric = "halstead.effort" -value = 66737.04848699087 +value = 70177.90308128356 [[entry]] path = "big-code-analysis-cli/src/thresholds.rs" @@ -845,11 +857,17 @@ qualified = "walk_seed" metric = "nargs" value = 5.0 +[[entry]] +path = "big-code-analysis-web/src/web/negotiate.rs" +qualified = "negotiate" +metric = "halstead.effort" +value = 48620.735506493904 + [[entry]] path = "big-code-analysis-web/src/web/server.rs" qualified = "run_parse" metric = "halstead.effort" -value = 82478.1543040951 +value = 84151.62410156948 [[entry]] path = "big-code-analysis-web/src/web/server.rs" @@ -1029,7 +1047,13 @@ value = 5.0 path = "src/metrics/loc/perl.rs" qualified = "PerlCode::compute" metric = "halstead.effort" -value = 51486.205050501565 +value = 51954.26146005158 + +[[entry]] +path = "src/metrics/npa/php.rs" +qualified = "PhpCode::compute" +metric = "halstead.effort" +value = 47869.643574826616 [[entry]] path = "src/metrics/npa/python.rs" @@ -1041,7 +1065,7 @@ value = 6.0 path = "src/ops.rs" qualified = "ops_inner" metric = "halstead.effort" -value = 67661.84803230887 +value = 70903.25755192844 [[entry]] path = "src/output/checkstyle.rs" @@ -1053,7 +1077,7 @@ value = 7.0 path = "src/output/code_climate.rs" qualified = "write_code_climate" metric = "halstead.effort" -value = 60596.13916454601 +value = 64499.43780660971 [[entry]] path = "src/output/dump.rs" @@ -1125,7 +1149,7 @@ value = 111.87939935484101 path = "src/output/funcspace_row.rs" qualified = "metric_values" metric = "halstead.effort" -value = 80351.11907066956 +value = 86304.11475310498 [[entry]] path = "src/spaces.rs" @@ -1155,7 +1179,7 @@ value = 5.0 path = "src/spaces/compute.rs" qualified = "compute_per_node" metric = "halstead.effort" -value = 63007.70094399036 +value = 63645.7536117776 [[entry]] path = "src/spaces/compute.rs" @@ -1167,7 +1191,7 @@ value = 7.0 path = "src/spaces/compute.rs" qualified = "metrics_inner" metric = "halstead.effort" -value = 116715.6053686016 +value = 121169.85621346734 [[entry]] path = "src/spaces/compute.rs" @@ -1185,7 +1209,19 @@ value = 6.0 path = "src/vcs/bus_factor.rs" qualified = "compute" metric = "halstead.effort" -value = 47694.803479864124 +value = 48322.36668354654 + +[[entry]] +path = "src/vcs/bus_factor.rs" +qualified = "greedy_truck_factor" +metric = "cognitive" +value = 15.0 + +[[entry]] +path = "src/vcs/bus_factor.rs" +qualified = "greedy_truck_factor" +metric = "halstead.effort" +value = 47756.87605789953 [[entry]] path = "src/vcs/bus_factor.rs" @@ -1221,7 +1257,7 @@ value = 6.0 path = "src/vcs/git/cached.rs" qualified = "build_cached" metric = "halstead.effort" -value = 74812.5424082547 +value = 78743.41070492903 [[entry]] path = "src/vcs/git/cached.rs" @@ -1251,13 +1287,13 @@ value = 6.0 path = "src/vcs/git/diff_parse.rs" qualified = "unquote_git_path" metric = "halstead.effort" -value = 71494.28221151498 +value = 74116.87497554444 [[entry]] path = "src/vcs/git/history.rs" qualified = "collect_events" metric = "halstead.effort" -value = 48928.29416311421 +value = 51486.680993977934 [[entry]] path = "src/vcs/git/history.rs" @@ -1319,6 +1355,12 @@ qualified = "compute_features" metric = "nexits" value = 5.0 +[[entry]] +path = "src/vcs/git/jit.rs" +qualified = "experience_features" +metric = "halstead.effort" +value = 49040.88359210979 + [[entry]] path = "src/vcs/git/jit.rs" qualified = "experience_features" @@ -1335,7 +1377,7 @@ value = 6.0 path = "src/vcs/git/jit.rs" qualified = "history_features" metric = "halstead.effort" -value = 52560.02538757721 +value = 54100.245911755286 [[entry]] path = "src/vcs/git/jit.rs" @@ -1373,6 +1415,12 @@ qualified = "build_trend" metric = "nexits" value = 5.0 +[[entry]] +path = "src/vcs/jit.rs" +qualified = "score" +metric = "halstead.effort" +value = 48585.950863463186 + [[entry]] path = "src/vcs/options.rs" qualified = "parse_iso8601" @@ -1391,23 +1439,29 @@ qualified = "fold_commit" metric = "nargs" value = 5.0 +[[entry]] +path = "src/vcs/replay.rs" +qualified = "replay" +metric = "halstead.effort" +value = 48730.801413215886 + [[entry]] path = "src/vcs/score.rs" qualified = "apply_percentile" metric = "halstead.effort" -value = 68450.87280037583 +value = 71452.77150113358 [[entry]] path = "src/vcs/stats.rs" qualified = "Accumulator::finalize" metric = "halstead.effort" -value = 79569.58843331039 +value = 89220.9266877338 [[entry]] path = "src/vcs/stats.rs" qualified = "Accumulator::record" metric = "halstead.effort" -value = 56240.27018612609 +value = 58878.05180751363 [[entry]] path = "src/vcs/trend.rs" @@ -1419,4 +1473,4 @@ value = 5.0 path = "src/wire.rs" qualified = "CodeMetrics::from" metric = "halstead.effort" -value = 49790.15156224738 +value = 55821.36704417831 diff --git a/.claude/rules/grammar-dispatch.md b/.claude/rules/grammar-dispatch.md index 370b21e0..f472c4a1 100644 --- a/.claude/rules/grammar-dispatch.md +++ b/.claude/rules/grammar-dispatch.md @@ -182,6 +182,19 @@ where the wrapper's span contains the leaf's, listing both is a no-op, because `LineSet::insert` / `insert_range` and `check_comment_ends_on_code_line` are all idempotent. +**Skip invented tokens by width, not by `is_missing()`.** Error recovery +inserts zero-width tokens for what the source lacks. It does so on +*valid* code the grammar cannot parse, as well as on invalid code. Some +grammars build these tokens without the MISSING flag: the C# and Elixir +zero-width identifiers are the known cases. So a guard meaning "this +node spells no written token" must test `start_byte() == end_byte()`, +and needs one malformed fixture per grammar it covers. + +Width is not a malformed-input test: valid trees have zero-width nodes +too, such as an empty Bash heredoc body. #1546 billed an operand spelled +`""` in all 23 languages, and its `is_missing()`-only fix still failed +C# and Elixir (lesson 95). + **Decide a grammar-reachable but language-invalid shape on what the change buys for *valid* input.** Grammars are routinely more permissive than their languages, so "the parser can produce this" does not settle diff --git a/.claude/rules/testing.md b/.claude/rules/testing.md index 0ede1d28..bb317cb9 100644 --- a/.claude/rules/testing.md +++ b/.claude/rules/testing.md @@ -118,6 +118,26 @@ equally common: two candidate rankings in #1465 were merely *different valid orderings* rather than defects, and the honest fix was to trim the test comment's claim. +**Re-run the perturbations when two predicates merge.** When two guards +are folded into one shared predicate, re-run the perturbation for every +test that guarded either one. A test can stop discriminating without +failing, because the merged paths can offset each other. During #1539, +once Perl ABC's `and` gate was shared, removing it left one slot row +reading 1: the slot dropped by one and the chain walk gained one. The +row's comment still claimed to guard `perl_condition_scores_itself`, +and only a re-run perturbation showed it no longer did. The comment was +corrected under #1539. + +**Run a perturbation sweep of an imported Python module with +`python3 -B`.** CPython reuses a cached `.pyc` when the source's mtime +(to the second) and size both match. A same-length edit such as `min` → +`max`, restored within the same second, therefore runs stale bytecode on +the next import. In #1562 that produced "isolated test fails, full run +passes" results that described the cache, not the code. Delete the +module's `__pycache__` before the sweep, then run every step with +`python3 -B` (or `PYTHONDONTWRITEBYTECODE=1`). `-B` stops new writes, but +a `.pyc` written before the sweep is still read. + After restoring, `git status` / `git diff --stat` must show exactly the edits you intend — nothing extra, nothing missing. diff --git a/.rustfmt-bail-baseline.txt b/.rustfmt-bail-baseline.txt index 9f2d36b9..3d8cc96e 100644 --- a/.rustfmt-bail-baseline.txt +++ b/.rustfmt-bail-baseline.txt @@ -122,27 +122,26 @@ # not checking it, and it does drift: before #1136 `src/getter/java.rs` # carried a 252-character match arm. -big-code-analysis-ast/src/getter.rs 3 big-code-analysis-ast/src/getter/bash.rs 3 big-code-analysis-ast/src/getter/c.rs 3 -big-code-analysis-ast/src/getter/cpp.rs 5 -big-code-analysis-ast/src/getter/csharp.rs 8 +big-code-analysis-ast/src/getter/cpp.rs 4 +big-code-analysis-ast/src/getter/csharp.rs 9 big-code-analysis-ast/src/getter/elixir.rs 3 big-code-analysis-ast/src/getter/go.rs 1 -big-code-analysis-ast/src/getter/groovy.rs 7 +big-code-analysis-ast/src/getter/groovy.rs 9 big-code-analysis-ast/src/getter/irules.rs 6 -big-code-analysis-ast/src/getter/java.rs 5 +big-code-analysis-ast/src/getter/java.rs 6 +big-code-analysis-ast/src/getter/js_family.rs 3 big-code-analysis-ast/src/getter/kotlin.rs 4 big-code-analysis-ast/src/getter/lua.rs 3 -big-code-analysis-ast/src/getter/mozcpp.rs 5 -big-code-analysis-ast/src/getter/perl.rs 3 +big-code-analysis-ast/src/getter/mozcpp.rs 4 +big-code-analysis-ast/src/getter/perl.rs 4 big-code-analysis-ast/src/getter/php.rs 11 big-code-analysis-ast/src/getter/python.rs 7 big-code-analysis-ast/src/getter/ruby.rs 4 big-code-analysis-ast/src/getter/tcl.rs 6 big-code-analysis-ast/src/macros/mod.rs 14 enums/src/macros.rs 1 -src/metrics/abc/elixir.rs 5 src/metrics/cognitive/perl.rs 8 src/metrics/cyclomatic.rs 7 src/metrics/cyclomatic/irules.rs 2 diff --git a/.snapshot-anchor-baseline.txt b/.snapshot-anchor-baseline.txt index 70540bf7..2dc4db08 100644 --- a/.snapshot-anchor-baseline.txt +++ b/.snapshot-anchor-baseline.txt @@ -9,7 +9,7 @@ # rather than a census of every scanned file (#1192 made the # scan recursive, which would otherwise have added 126 zeroes). -src/metrics/abc.rs 27 +src/metrics/abc.rs 26 src/metrics/cyclomatic.rs 1 src/metrics/npa.rs 12 src/metrics/npm.rs 23 diff --git a/AGENTS.md b/AGENTS.md index 4cedaa9f..ff2204ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -568,6 +568,23 @@ metric-value-only (no structural changes). Run `cargo insta test one at a time can shift `assertion_line` fields, causing a cascade where previously-matching snapshots become stale. +Three traps in that loop: + +- **The corpus runner aborts after its first panics**, so a plain run + reports only part of what moved. #1296 showed 16 of 254 files. + Collect with `INSTA_FORCE_PASS=1 cargo test --no-fail-fast` before + counting or reviewing. +- **A forced pass also hides inline snapshots.** An + `assert_json_snapshot!(…, @"…")` that moved passes and leaves only a + `src/**/.*.pending-snap` file, not a `.snap.new`. Review both kinds + (#1552). +- **`cargo insta test --accept` accepts every pending snapshot it finds + in the workspace**, not only the ones its own run produced. That + includes corpus snapshots left by an earlier forced run (#1486). The + per-test-file advice above holds only when nothing else is pending. + Otherwise review everything that is pending, or accept a filtered set + with `cargo insta accept --snapshot `. + **Anchor every `insta::assert_json_snapshot!` call.** A bare `insta::assert_json_snapshot!(metric.X)` records whatever production emitted at acceptance time — including bugs (see issue #95 and lesson 2 diff --git a/CHANGELOG.md b/CHANGELOG.md index d7cae698..4f98958b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,94 @@ cognitive; the word-operator entry (#1530) moves cognitive only; and the Elixir clause entry (#1531) moves cyclomatic for a typespec function type and a keyword-form `cond` catch-all. +**Halstead drift.** The entries under **Changed** apply #1395's rule that +punctuation and declaration keywords are vocabulary to the languages +that still left them out, so Halstead and the Maintainability Index +move for most Python, Rust, C, C++, Java, JavaScript, TypeScript, +Kotlin, Groovy and C# code. Python and Rust rise most, from #1486. The +generic `<>` pair (#1559) lowers effort for most code that uses +generics. Several **Fixed** entries also move only Halstead and are +marked as such. If you gate `halstead.effort` or `mi.*` with `bca +check`, expect new offenders in Python, Rust, Java, C# and C++ code, +and refresh `.bca-baseline.toml`. The `bca init` default for +`halstead.effort` and the book's per-language calibration table were +derived before these changes. Their re-derivation is tracked in #1569. + +### Added + +- **`Getter::get_operator_spelling`** (#1559), a defaulted trait method + in `big-code-analysis-ast` that lets a getter bill an operator under a + fixed text key when its grammar gives two operators the same kind id. + It is additive: existing implementations compile unchanged. Like the + rest of `big-code-analysis-ast`, it is outside the stability contract + (see `STABILITY.md`). Nothing new reaches the `big-code-analysis` + crate's public surface. + +### Changed + +- **Python and Rust bill their punctuation as Halstead operators** + (#1486). Python bills `()`, `[]` and `{}` once per pair, plus `;` and + every syntactic `:`. Rust bills `:` as an entry separate from `::`. An + f-string's `{` and format-spec `:` stay unbilled. Every Python and Rust + Halstead score rises and every MI score falls; on DeepSpeech's Python, + N1 rises by about 47%. +- **Type-declaration keywords are Halstead operators** in Java (`class`, + `interface`, `enum`, `record`, `@interface`), JavaScript and TypeScript + (`class`; TypeScript also `interface`, `enum`), Groovy (`class`, + `interface`, `enum`, `trait`, `record`), C and C++ (`struct`, `union`, + `enum`, C++ `class`), Objective-C (`@interface`, `@implementation`, + `@protocol`) and Python (`class`) (#1550, #1552). These languages now + match C#, Kotlin, PHP, Ruby and Rust. A `class` that declares nothing + (Java `C.class`) is not billed. +- **C++ and TypeScript declaration keywords and TypeScript type names** + (#1557). C++ bills `namespace`, `template`, `typename`, `typedef` and + `using`; C and Objective-C bill `typedef`. TypeScript bills `type`, + `namespace`, `module`, `declare` and `global`. Every TypeScript + `type_identifier`, including alias and interface names, annotations + and generic arguments, is now an operand. `template ` and + `template ` bill alike. A binding named `type` in an export + clause (`export { type }`) is not billed as the keyword. +- **Java type names are Halstead operands** (#1560), as in C#, Rust and + TypeScript. A class literal's interior and the inferred `var` stay + unbilled. +- **Kotlin bills its declaration modifiers and `interface`** (#1556, + #1558): `enum`, `data`, `sealed`, the visibility keywords, `open`, + `override`, `suspend`, `reified`, variance `out` and the rest. Before + this, only `class` and `object` were billed. The same words used as + names bill no operator, including where the grammar fails to parse + the name (`open = !open`, `val suspend = 1`, a parameter named + `vararg`). +- **TypeScript bills `satisfies`, `keyof`, `infer`, `is` and + `asserts`** (#1561). +- **Generic angle brackets bill as one `<>` pair** (#1559) in Rust (type + arguments and parameters, turbofish, `for<'a>`, `use`), Java, + Groovy, C#, TypeScript, TSX, Kotlin, C++ templates and Objective-C + (lightweight generics, protocol lists and `id

`). The opening `<` + bills `<>` and the closing `>` bills nothing, the same convention as + `()`, `[]` and `{}`. Comparisons and shifts are unchanged, with two + exceptions where the grammar reads a comparison as a generic list. + The first is a C++ comparison that tree-sitter-cpp reads as a template + argument list (`x = a < b || c > (d)`). The second is a C# call + argument pair spelled with `>=` (`F(a < b, c >= d)`), which the + grammar parses as an assignment. Both bill as `<>`. N1 falls + for code that uses generics, and effort fell in about 87% of the moved + C++ and Rust corpus files. +- **Perl and PHP `xor` scores as a value, not a decision** (#1536), like + Java's `^` and the eager `xor` of C++ and Kotlin. It adds no + cyclomatic or cognitive increment, and the boolean slot holding it + pays one ABC condition. PHP `xor` moves in all three metrics. +- **Rust `let`-`else` is a decision** (#1542, #1548). It adds one + cyclomatic decision (standard and modified), and cognitive scores it + +1 plus nesting, the same as its `if let` twin. Before, cyclomatic + ignored it and cognitive gave a flat +1 through the `else` token. +- **Elixir `with` clauses are decisions** (#1535). Each `pattern <- expr` + adds one cyclomatic decision and one ABC condition, like Rust's `if + let`, and a `when` guard on the pattern counts as a guard. Modified + cyclomatic no longer charges `with` a flat +1; its `else` adds one only + when it holds a clause other than `_`. A `for … reduce:` accumulator + clause follows the multi-clause `fn` rule, so a single clause is no + longer a decision. + ### Fixed - **A C++ or Mozcpp comparison-operator overload no longer scores a @@ -199,6 +287,112 @@ type and a keyword-form `cond` catch-all. `and` now keys to `&&` and `or` to `||`, as C++ does since #1522. Perl's low-precedence `and`, which cognitive never counted because the grammar parses it as a `unary_expression`, now scores like `&&`. +- **A negated C# call argument scores a condition** (#1537). `G(!a)`, + `G(name: !a)`, `new B(!a)`, `: base(!a)` and primary-constructor base + calls scored 0, where Java's `g(!a)` scores 1, because the `argument` + wrapper was never peeled. +- **Perl comparisons no longer over-count a condition when the grammar + mis-binds precedence** (#1273). tree-sitter-perl 1.1.2 parses + `$a > $b ? 1 : 2` as `$a > ($b ? 1 : 2)`, and `$a > $b && $c` the same + way. Each misparsed chaining comparison (`< > <= >= lt gt le ge == != + eq ne`) added one ABC condition. They now score like their + parenthesised and JavaScript twins. This is a local workaround for an + upstream grammar defect. +- **Perl auto-quoted hash keys no longer score as operators** (#1539, + #1541, #1545). tree-sitter-perl 1.1.2 misparses bareword keys: + - `and` (`$h{and}`, `(and => 1)`) scored a decision in cyclomatic, + cognitive and ABC. + - `not` was billed as a Halstead operator and peeled by ABC as a + negation. + - A `-bareword` key (`$h{-foo}`, `(-text => 1)`) was read as a file + test, which billed a fragment of the word (`oo`) and an ABC branch. + Each key now scores like its quoted or `or` twin, and real operators + and file tests are unchanged. ABC still reads the value after an + `and`, `not`, `-and` or `-not` key, so `f(and => !$a)` pays the + negation's condition. A `-and` / `-not` key keeps its sign however it + is spaced (`(- not => 1)`), and a real subtraction before a key + (`(1 - and => 2)`) keeps its `-`. A `-bareword` key whose letter is + no file-test letter (`-name`) still scores as a negated bareword. +- **Perl `$x =~ s///` and `tr///` no longer score a condition by use** + (#1540), matching bare `s///`. A substitution is an edit, not a test. + Inside a boolean slot it still counts once, and pattern matches are + unchanged. +- **Perl Halstead no longer bills comment text** (#1549). The `#` of + every comment was billed as an operator, and so were the `#`, `[`, `]` + and `\` inside a bare `/…/` pattern, which now scores as `m//` and + `qr//` do. Halstead only. +- **Bash `-a` / `-o` and `=` score like their `&&` / `==` twins** + (#1536). `[ X -a Y ]`, `[ X ] && [ Y ]` and `[[ X && Y ]]` now agree + in cyclomatic, ABC and cognitive. `-a` / `-o` are logical connectives, + not ABC comparisons, and `=` inside a test is an ABC comparison. It + stays an assignment in `(( … ))`. A run of `=` or `=~` tests joined by + one connective, which the grammar nests around the connectives, is one + cognitive sequence, like its `==` twin. +- **An Elixir named operator is billed once** (#1534). `&==/2`, + `&and/2` and `Kernel.||(a, b)` billed both an operand and an operator, + and a captured `&&` / `||` / `and` / `or` (`&||/2`) added a cyclomatic + decision. A remote call on `Kernel` (`Kernel.||(a, b)`) applies the + operator and keeps its decision. The same call on a user module + (`Foo.||(a, b)`) is an ordinary function call and decides nothing. +- **C# lambda modifiers and implicit parameters are billed** (#1482, + #1544). A lambda's or anonymous method's `static` / `async` is an + operator, keyed as the declaration spelling is. The bare parameter of + `x => x` is an operand, as in `(x) => x`. Halstead only. +- **Overloaded operator names are Halstead operators** (#1296). This + covers C# `operator true` / `operator false` (previously operands), + and C++ / Mozcpp `operator[]`, `operator()`, `operator""_x`, + `operator co_await` and the `operator` keyword (previously billed + nowhere); `bca ops` lists the C++ names as `operator[]` and + `operator()`, apart from a subscript's `[]` and a call's `()`. So are + `co_await` in an expression and the `co_return` and `co_yield` + statements, which NExits now counts as exits (#1547). ABC reads a + `co_return` value as it reads a `return` value, so `co_return !x` pays + one condition. +- **JSX element text is a Halstead operand** (#1483), keyed on its + trimmed text, in JavaScript, Mozjs and TSX. Whitespace-only text is + ignored, and a character reference such as `&` in element content + is its own operand. +- **A JavaScript function expression bills `function` once** (#1554), + not twice. This applies to JavaScript, Mozjs, TypeScript and TSX. +- **Python f-string `=` is not an assignment** (#1551). The + self-documenting `=` of `f"{x=}"` is no longer billed. Halstead only. +- **Halstead skips zero-width nodes from error recovery** (#1546). An + inserted identifier billed an operand spelled `""` in every language, + and an inserted `;` / `)` billed a second operator. +- **A bodiless C++ specifier opens no space** (#1555). Forward + declarations, elaborated type specifiers (`struct S *p;`, parameter + types, casts) and explicit template instantiations opened phantom + struct/class spaces. In DeepSpeech, 4,452 such spaces disappear. This + lowers `cyclomatic.sum` by one base per removed space. +- **`make check-test-lang-gates` catches a helper gated out of a build + its caller compiles in** (#1562), when the caller names its parsers + only inside an inner `#[cfg]` block. A `const` fixture table passed + only to such calls is checked the same way. +- **Perl's fat comma lists as `=>` in `bca ops`**, not as the grammar's + `fat_comma` node name. Rendering only; no count moves. +- **A trailing Perl `,` or `=>` in a condition adds no condition** + (#1526). `if ($a > 1,)` and `if ($a > 1 =>)` scored 2 against 1 for + `if ($a > 1)`, which Perl reduces them to: the grammar names its + separators, and the trailing one was read as the list's value. +- **A Perl pattern keeps its role inside parentheses** (#1467, #1540). + `$x =~ (/foo/)` and `split((/foo/), $x)` scored the match their bare + forms do not, and `$x =~ (s/a/b/)` scored the `=~` that + `$x =~ s/a/b/` does not. `/foo/ =~ $x`, where the pattern matches + `$_` before its result is bound, now scores 2 rather than 1. +- **Only Perl's builtin `split` exempts its delimiter** (#1467). + `My::split(/foo/, $x)` scored 0; it now scores the match, as + `My::other(/foo/, $x)` does. `CORE::split` is still the builtin. +- **A Kotlin parameter named `vararg` stays a name behind a comment** + (#1558). `fun f(vararg /*note*/: Int)` billed a `vararg` operator + that `fun f(vararg: Int)` does not. +- **A Perl file test before a commented or line-broken `=>` stays a + file test** (#1545). Perl quotes `-f` before `=>` only across spaces + and tabs, so `(-f # c⏎ => 7)` and `(-f⏎ => 7)` run as `(-f, 7)`, but + both billed a `-f` key operand. +- **Wide Perl and Kotlin lists no longer make the walk quadratic** + (#1545, #1558, #1467). A call with thousands of `-foo => …` keys, a + function with thousands of modified parameters, and a list of + thousands of patterns each took seconds; all three are now linear. ## [2.3.0] - 2026-10-07 diff --git a/big-code-analysis-ast/src/checker.rs b/big-code-analysis-ast/src/checker.rs index 88337697..02e3abe9 100644 --- a/big-code-analysis-ast/src/checker.rs +++ b/big-code-analysis-ast/src/checker.rs @@ -818,6 +818,19 @@ pub fn csharp_member_has_accessors(node: &Node) -> bool { csharp_accessor_count(node) > 0 } +/// Returns `true` when a C++ `struct_specifier` / `class_specifier` +/// defines a type body (`struct S { … }`) rather than only naming one. +/// A forward declaration (`struct S;`) and an elaborated type specifier +/// (`struct S *p;`, the parameter type in `void f(struct S *q)`) carry no +/// `body` field. +/// +/// Shared by the C++ and Mozcpp `is_func_space` and `get_space_kind`, so +/// a bodiless specifier neither opens a space nor is labelled one (#1555). +#[must_use] +pub(crate) fn cpp_specifier_has_body(node: &Node) -> bool { + node.child_by_field_name("body").is_some() +} + /// Whether `node` is a bare keyword token the C# grammar *aliased* to /// `modifier`, rather than the `modifier` rule itself. /// @@ -829,11 +842,8 @@ pub fn csharp_member_has_accessors(node: &Node) -> bool { /// - `_parameter_type_with_modifiers` — `this`, `scoped`, `ref`, `out`, /// `in`, `readonly` in parameter position, the six #1418 bills. /// - `_lambda_expression_init` and `anonymous_method_expression` — -/// `static` and `async` on a lambda or `delegate { … }`. These reach -/// [`CsharpCode::get_op_type_with_code`]'s fallthrough and stay -/// unclassified, exactly as they were before #1418; billing them is a -/// separate decision, since the *declaration* spelling of the same two -/// keywords is billed through its leaf. +/// `static` and `async` on a lambda or `delegate { … }`, billed as +/// operators like their declaration spelling (#1482). /// /// Child-presence is therefore what separates an alias from the rule, /// and it is the grammar's shape rather than an inference about a broken @@ -3388,6 +3398,7 @@ mod tests { /// Also asserts the two spellings agree off a known chain and off a /// `Node::parent` climb: the walk now threads a chain, and the /// answer must not depend on which it gets. + #[cfg(any(feature = "irules", feature = "tcl"))] fn braced_word_string_verdicts( label: &str, code: &[u8], diff --git a/big-code-analysis-ast/src/checker/cpp.rs b/big-code-analysis-ast/src/checker/cpp.rs index 7c48d33e..4539bb24 100644 --- a/big-code-analysis-ast/src/checker/cpp.rs +++ b/big-code-analysis-ast/src/checker/cpp.rs @@ -21,18 +21,22 @@ impl Checker for CppCode { // will silently drop those nodes from FuncSpace creation the next // time a grammar bump starts emitting them (see lesson 2 in // `docs/development/lessons_learned.md`). + // + // A struct / class specifier opens a space only when it has a body, + // gated by the same predicate as `get_space_kind` (#1555). fn is_func_space(node: &Node) -> bool { - matches!( - node.kind_id().into(), - Cpp::TranslationUnit - | Cpp::FunctionDefinition - | Cpp::FunctionDefinition2 - | Cpp::FunctionDefinition3 - | Cpp::FunctionDefinition4 - | Cpp::StructSpecifier - | Cpp::ClassSpecifier - | Cpp::NamespaceDefinition - ) + match node.kind_id().into() { + Cpp::StructSpecifier | Cpp::ClassSpecifier => cpp_specifier_has_body(node), + kind => matches!( + kind, + Cpp::TranslationUnit + | Cpp::FunctionDefinition + | Cpp::FunctionDefinition2 + | Cpp::FunctionDefinition3 + | Cpp::FunctionDefinition4 + | Cpp::NamespaceDefinition + ), + } } // Issue #285 contract: keep this in sync with `is_func_space` and diff --git a/big-code-analysis-ast/src/checker/csharp.rs b/big-code-analysis-ast/src/checker/csharp.rs index 23c385ff..de540d54 100644 --- a/big-code-analysis-ast/src/checker/csharp.rs +++ b/big-code-analysis-ast/src/checker/csharp.rs @@ -118,32 +118,28 @@ impl Checker for CsharpCode { // them through the lexeme-keyed `primitive_operators` map so // distinct keywords count as distinct operators (issue #286). // - // The five parameter-modifier keywords join it for the mirror - // reason (#1418). Each has *two* kind spellings for one - // operator: its own token, and the childless `modifier` the - // grammar aliases it to in parameter position. Keying only the - // alias by text would split one `ref` across the two maps and - // count it twice in `n1` — the #453 shape — while keying the - // alias by `kind_id` would instead collapse all five spellings - // into one operator that `bca ops` renders as the word - // `modifier`, which appears nowhere in the source. Text is the - // only key under which the two positions are the same operator. + // Every keyword the grammar aliases to a childless `modifier` + // joins it for the mirror reason: the five parameter modifiers + // (#1418) and a lambda's `static` / `async` (#1482). Each has + // *two* kind spellings for one operator: its own token, and the + // aliased `modifier`. Keying only the alias by text would split + // one `ref` or `static` across the two maps and count it twice + // in `n1` — the #453 shape — while keying the alias by `kind_id` + // would instead collapse every spelling into one operator that + // `bca ops` renders as the word `modifier`, which appears + // nowhere in the source. Text is the only key under which the + // two positions are the same operator, so the bare kind of every + // keyword the getter bills through the alias must be listed too. // - // Listing the five bare kinds moves no existing count: each - // renders exactly one text through `get_operator_id_as_str` and - // no other kind renders that text, so the vocabulary entry only - // changes which map holds it. + // Listing the bare kinds moves no existing count: each renders + // exactly one text through `get_operator_id_as_str` and no other + // kind renders that text, so the vocabulary entry only changes + // which map holds it. // - // The shared predicate is wider than those five, and deliberately - // so: it holds for every aliased `modifier`, including the `this` - // receiver that `get_op_type_with_code` bills as an operand and - // the `static` / `async` of a lambda that it leaves unclassified. - // Neither is a disagreement between the two methods, because - // `compute_halstead` consults this one only under its `Operator` - // arm — so the answer is never read for anything the getter did - // not call an operator. Widening the getter later (billing a - // lambda's `static`) therefore needs no change here, and gets the - // lexeme key it would want. + // The shared predicate also holds for the aliased `this` + // receiver, which `get_op_type_with_code` bills as an operand. + // That is no disagreement, because `compute_halstead` consults + // this method only under its `Operator` arm. matches!( node.kind_id().into(), Csharp::PredefinedType @@ -152,6 +148,8 @@ impl Checker for CsharpCode { | Csharp::In | Csharp::Readonly | Csharp::Scoped + | Csharp::Static + | Csharp::Async ) || csharp_is_aliased_modifier(node) } } diff --git a/big-code-analysis-ast/src/checker/kotlin.rs b/big-code-analysis-ast/src/checker/kotlin.rs index 93f18636..ebab1f32 100644 --- a/big-code-analysis-ast/src/checker/kotlin.rs +++ b/big-code-analysis-ast/src/checker/kotlin.rs @@ -75,4 +75,15 @@ impl Checker for KotlinCode { // followed by an `if_expression` (not an `if_statement`, and no // wrapping clause node). impl_is_else_if_prev_sibling!(Kotlin, IfExpression, Else); + + // `reified` is the one modifier the grammar spells as an aliased + // named kind, `reification_modifier`, rather than a keyword token. + // Keying it by `kind_id` would make `bca ops` list an operator + // named `reification_modifier`, which appears nowhere in the + // source; keying it by text lists `reified` (#1558). It has no + // other kind, so nothing splits across the two `n1` maps. + #[inline] + fn is_primitive(node: &Node) -> bool { + node.kind_id() == Kotlin::ReificationModifier + } } diff --git a/big-code-analysis-ast/src/checker/mozcpp.rs b/big-code-analysis-ast/src/checker/mozcpp.rs index 5d681996..7f46535d 100644 --- a/big-code-analysis-ast/src/checker/mozcpp.rs +++ b/big-code-analysis-ast/src/checker/mozcpp.rs @@ -21,18 +21,22 @@ impl Checker for MozcppCode { // will silently drop those nodes from FuncSpace creation the next // time a grammar bump starts emitting them (see lesson 2 in // `docs/development/lessons_learned.md`). + // + // A struct / class specifier opens a space only when it has a body, + // gated by the same predicate as `get_space_kind` (#1555). fn is_func_space(node: &Node) -> bool { - matches!( - node.kind_id().into(), - Mozcpp::TranslationUnit - | Mozcpp::FunctionDefinition - | Mozcpp::FunctionDefinition2 - | Mozcpp::FunctionDefinition3 - | Mozcpp::FunctionDefinition4 - | Mozcpp::StructSpecifier - | Mozcpp::ClassSpecifier - | Mozcpp::NamespaceDefinition - ) + match node.kind_id().into() { + Mozcpp::StructSpecifier | Mozcpp::ClassSpecifier => cpp_specifier_has_body(node), + kind => matches!( + kind, + Mozcpp::TranslationUnit + | Mozcpp::FunctionDefinition + | Mozcpp::FunctionDefinition2 + | Mozcpp::FunctionDefinition3 + | Mozcpp::FunctionDefinition4 + | Mozcpp::NamespaceDefinition + ), + } } // Issue #285 contract: keep this in sync with `is_func_space` and diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index 34f6d3f7..85278aae 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -17,6 +17,8 @@ use crate::lang_helpers::tcl_family::{ }; use crate::space_kind::SpaceKind; use crate::traits::Search; +use generic_angles::{GenericAngleKinds, never_misparsed}; +use js_family::impl_js_family_get_op_type; use crate::*; @@ -66,208 +68,6 @@ macro_rules! get_operator { }; } -// Emit a `Getter::get_op_type` body for a JS-family language. The four -// JS-family grammars (JavaScript, MozJS, TypeScript, TSX) share most of -// their Halstead operator/operand kind classifications; per-language -// deltas are passed as bracketed extras so all four impls stay in -// lockstep when a kind is added or removed (issue #299). -// -// `$op_extras` per language: -// * JavaScript / MozJS: `OptionalChain` — the bare `?.` token (these -// grammars expose no `optional_chain` wrapper). -// * TypeScript / TSX: `QMARKDOT`, `PredefinedType` — `QMARKDOT` is -// the bare `?.` token under the `optional_chain` wrapper (issue -// #281); `PredefinedType` is the TS type keyword set (`string`, -// `number`, `boolean`, …). -// * JavaScript / MozJS / TSX: `LTSLASH`, `SLASHGT` — the JSX closing -// and self-closing tag delimiters, which these grammars spell as -// their own two-character tokens rather than as a `<` / `>` pair. -// Until #1395 neither was in either arm, so a JSX element billed -// its opening `<` and its `>`s but nothing for `` -// reported one bracket operator where the source spells two. They -// are extras rather than members of the shared arm because -// TypeScript is the one grammar of the four with no JSX and -// therefore no such variants to name. -// -// `$operand_extras` per language: -// * JavaScript / MozJS / TSX: `Identifier2`, `String2` — anonymous -// keyword aliases the JS grammar exposes for `Identifier` and -// `String`. TSX's `String2` (kind_id 261) is its string-literal -// alias, so it stays an operand like JS's. -// * TypeScript: none. TS's own aliases are either operators or -// deliberately unclassified (below). -// The `: string` type-keyword aliases (TS `String2`, kind_id 135; -// TSX `String3`, kind_id 141) are deliberately absent: they are -// emitted only as the child of a `predefined_type` wrapper, which -// already contributes the text-keyed `"string"` operator, and -// #313's listing of the keyword child as an operand counted one -// source token twice (#1261, which also narrowed -// `Checker::is_string` to match). -// -// **A member access contributes its leaves, never the composite.** -// `a.b` is the operands `a` and `b` plus the `.` operator — three -// vocabulary entries, not four. Until #1263 the operand arm listed the -// `member_expression` wrapper *and* the walker descended into its -// `object` / `property` children, so every access billed one extra -// `N2` entry keyed on the whole `a.b` text (and `a.b.c` billed two: -// `a.b` and `a.b.c`). The composite is what the rest of the workspace -// already excludes — C, C++, Java, Rust, Python, Go, Kotlin, Ruby, Lua -// and PHP were all measured leaves-only for the identical shape — so -// identical code scored differently by language. The same call applies -// to TS's `nested_identifier` (`namespace N.M`), C#'s -// `qualified_name` / `generic_name` / `alias_qualified_name`, and -// Groovy's `qualified_name` / `qualified_type`. -// -// This is grammar-dispatch section 5's "any predicate listing both a -// container and a kind it can contain double-counts", and its section -// 6 corollary is why `PrivatePropertyIdentifier` joins the operand arm -// in the same change: `this.#x`'s `#x` leaf was in no operand list, so -// the composite had been its only count, and a bare deletion would -// have regressed private-field access to zero operands. (The `#x` in -// the *field definition* `#x = 1` had never been counted at all — no -// wrapper covered it — so this fixes that half too.) -// -// The `TemplateString` interpolation guard is shared verbatim (issue -// #192): a bare `` `...` `` mirrors a `"..."` operand, but an -// interpolated template must yield `Unknown` because its inner -// `TemplateSubstitution` expressions are walked separately. -// -// The `Regex` arms are shared the same way (issue #1314), and for the -// same reason: all four grammars spell a regex literal identically — -// a `regex` wrapper whose two delimiters are `SLASH`, the kind id real -// division uses — so neither arm has a per-language delta to hoist to -// a call site. Only the ids differ (`SLASH` 87/81/90/87, `Regex` -// 224/250/264/225) and the macro names both through the enum, so no -// invocation mentions either. See the arms themselves for the -// fabrication and the missing-operand halves. -macro_rules! impl_js_family_get_op_type { - ( - $lang:ident, - op_extras: [$($op_extra:ident),* $(,)?], - operand_extras: [$($operand_extra:ident),* $(,)?] - $(, predefined_void: $predefined_type:ident)? $(,)? - ) => { - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { - use $lang::*; - - // TS/TSX only: a `void` return / parameter type is parsed as a - // `predefined_type` wrapper around an inner `void` token. Both - // the wrapper (routed through `is_primitive` into the text-keyed - // `primitive_operators` map as `"void"`) and the inner `Void` - // token (a standalone expression operator, e.g. `void 0`) would - // otherwise classify as operators, double-counting one source - // `void` as two Halstead operators (issue #453). Only `void` has - // an operator-kind child; the `string` keyword's child is an - // operand-kind alias, which collided in the other direction - // (operator + operand) until #1261 dropped it from the operand - // extras. Suppress the wrapper here and - // let the inner `Void` token carry the single operator, keeping - // the kind_id-keyed count consistent with expression `void 0` - // (the lesson-4 `n1 == dedupe(ops.operators)` invariant). - $( - if node.kind_id() == $predefined_type as u16 - && node - .child(0) - .is_some_and(|child| child.kind_id() == Void as u16) - { - return TokenRole::Unknown; - } - )? - - match node.kind_id().into() { - // Regex delimiter punctuation. A `regex` literal spells - // both of its delimiters with the same kind id real - // division uses, so `const a = /abc/g;` reported a `/` - // operator with no division in the source and n1/N1 - // counted the literal's punctuation as arithmetic - // (#1314, the JS-family sibling of Elixir #1256 and - // Ruby/Perl #1312). The `Regex` node itself is the - // operand (below), so the delimiters are suppressed - // exactly when their parent is that node — the - // compound-leaf guard of grammar-dispatch section 5. - // - // Parent, not ancestor, for correctness by - // construction rather than by observation: unlike - // Ruby's `#{…}`, a JS regex admits no nested - // expression at all (`regex_pattern` and `regex_flags` - // are leaves), so no fixture can tell this guard from - // an ancestor-scanning one. The halstead test - // `js_regex_delimiter_guard_is_parent_scoped_is_unobservable` - // records that, and pins the grammar property it rests - // on, rather than implying coverage the suite lacks. - // - // `SLASH2` — the aliased regex-start token every one of - // these grammars carries — needs no arm: it is absent - // from the operator list below, so it already lands on - // `Unknown`, which is what a delimiter should be. That - // is the one place this differs from Ruby's guard, - // where `SLASH2` had to be moved off the arithmetic arm. - SLASH - if ancestors.parent_has_kind(node, Regex as u16) => - { - TokenRole::Unknown - } - Export | Import | Import2 | Extends | DOT | From | LPAREN | COMMA | As | STAR - | GTGT | GTGTGT | COLON | Return | Delete | Throw | Break | Continue | If - | Else | Switch | Case | Default | Async | Do | For | In | Of | While | Try - | Catch | Finally | With | EQ | AT | AMPAMP | PIPEPIPE | PLUS | DASH | DASHDASH - | PLUSPLUS | SLASH | PERCENT | STARSTAR | PIPE | AMP | LTLT | TILDE | LT | LTEQ - | EQEQ | BANGEQ | GTEQ | GT | PLUSEQ | BANG | BANGEQEQ | EQEQEQ | DASHEQ - | STAREQ | SLASHEQ | PERCENTEQ | STARSTAREQ | GTGTEQ | GTGTGTEQ | LTLTEQ | AMPEQ - | CARET | CARETEQ | PIPEEQ | Yield | LBRACK | LBRACE | Await | QMARK - | QMARKQMARK | EQGT | DOTDOTDOT | New | Let | Var | Const | Function - | FunctionExpression | SEMI | Typeof | Instanceof | Void - // `get`/`set` accessor keywords are operators, matching the - // C# getter's `Get | Set | Init | Add | Remove` accessor arm. - // Before #695 the JS family classified them as operands, so - // the same accessor keyword landed in opposite Halstead - // groups across languages, skewing n1/n2 for accessor-heavy - // code (#695). - | Set | Get - $(| $op_extra)* => TokenRole::Operator, - // `Regex` is the literal's own node and contributes one - // operand, the way Ruby's `Regex` and Elixir's `Sigil` - // do. It was in neither arm before #1314, so `/abc/g` - // reached the vocabulary from *neither* side: a - // fabricated `/` operator and no operand at all. Its - // `regex_pattern` / `regex_flags` children stay - // unclassified, so the wrapper cannot double-count - // them, and no interpolation is possible inside a JS - // regex — hence a plain arm here rather than the - // `string_operand_type` dispatch the template literal - // below needs. - // `PrivatePropertyIdentifier` is the `#x` leaf of a - // private class field, in both its declaration - // (`#x = 1`) and its access (`this.#x`). See the - // leaves-not-composites note above the macro for why - // it had to be added in the same change that dropped - // `MemberExpression*`. - // - // `MetaProperty` is `import.meta` / `new.target`: one - // atomic operand, like `this`, whose `meta` / `target` - // leaves are anonymous tokens no arm classifies. It is - // the one composite the #1263 drop has to keep - // (grammar-dispatch §6) — without it the meta-object - // contributes no operand at all while `this.env.x` - // still yields three. - Identifier | PropertyIdentifier | PrivatePropertyIdentifier | MetaProperty - | String | Number | True | False | Null | This | Super | Undefined | Regex - $(| $operand_extra)* => TokenRole::Operand, - // A `` `...` `` is a string literal; without interpolation it - // mirrors `"..."` and contributes one operand. When it has a - // `TemplateSubstitution` child the inner expression is already - // walked and classified, so counting the wrapper too would - // double-count its contribution to `N2` (issue #192, same - // pattern as #183 C# / #191 Kotlin / #199 Perl). - TemplateString => { - Self::string_operand_type(node, &[TemplateSubstitution as u16]) - } - _ => TokenRole::Unknown, - } - } - }; -} - /// The default space name: the node's `name` field, else ``. /// /// A free function as well as the trait default so a language that needs @@ -434,11 +234,9 @@ pub trait Getter { /// opens a JSX tag, brackets a Lua `` attribute, names a C# /// `operator <` overload, disambiguates a Kotlin `super`, or /// delimits a Perl `` readline. Every impl that classifies - /// punctuation at all already bills `(`, `{`, `[`, `,`, `;`, `.` - /// and `:` on exactly that reading, for each glyph its language - /// spells — bar two, which #1395 did not revisit: Python - /// classifies only `,` and `.` of that set, and Rust bills `::` - /// but not the type-annotation `:`. + /// punctuation at all bills `(`, `{`, `[`, `,`, `;`, `.` and `:` on + /// exactly that reading, for each glyph its language spells. Python + /// and Rust were the last two exceptions, closed by #1486. /// /// A *decision* metric gates the same token by role, and is /// supposed to disagree: ABC's `conditions` counts a `<` only under @@ -476,7 +274,10 @@ pub trait Getter { /// a script-taking command. /// /// A bracket delimiting *syntax* fits neither shape: no literal - /// would absorb its contribution, so it stays an operator. + /// would absorb its contribution, so it stays an operator. A + /// generic list's `<` / `>` is billed like the other bracket pairs, + /// once as `<>` on the opener ([`Self::get_operator_spelling`], #1559), + /// which leaves the comparison `<` and `>` their own entries. /// /// `ancestors` is the chain the walker descended through. Most /// impls read a parent from it to disambiguate a token whose role @@ -516,6 +317,22 @@ pub trait Getter { Self::get_op_type(node, ancestors) } + /// The fixed spelling an operator `node` is keyed under in place of + /// its kind id, for a token whose kind the grammar shares with a + /// different operator; `None` keys it by kind. A generic list's `<` + /// is the case (#1559): it has the comparison's kind but is keyed as + /// the `"<>"` pair, as `()`, `[]` and `{}` are. Consulted only for a + /// node [`get_op_type_with_code`](Self::get_op_type_with_code) calls + /// an operator. + #[inline] + #[must_use] + fn get_operator_spelling<'a>( + _node: &Node<'a>, + _ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + None + } + /// Returns the source-byte slice used to key a Halstead *operand*. /// The default keys on the operand node's full byte range. Kotlin /// overrides this to narrow a short-interpolation name token @@ -1127,11 +944,13 @@ mod ccomment; mod cpp; mod csharp; mod elixir; +mod generic_angles; mod go; mod groovy; mod irules; mod java; mod javascript; +mod js_family; mod kotlin; mod lua; mod mozcpp; diff --git a/big-code-analysis-ast/src/getter/c.rs b/big-code-analysis-ast/src/getter/c.rs index 5c3e0a77..8fb6520b 100644 --- a/big-code-analysis-ast/src/getter/c.rs +++ b/big-code-analysis-ast/src/getter/c.rs @@ -80,7 +80,13 @@ impl Getter for CCode { // modifier has a distinct kind_id, so keying by kind_id (the default // `operators` store) keeps them distinct in n1 while `long long`'s // two `long` tokens correctly fold to one n1 entry but two N1 hits. - | Signed | Unsigned | Long | Short => TokenRole::Operator, + | Signed | Unsigned | Long | Short + // `struct` / `union` / `enum`, which C++ bills since #1552, + // and `typedef` since #1557. They are C keywords, not + // C++-only forms, so leaving them out here would score a + // header differently by the grammar that parsed it. The + // derivation is on the Cpp arm. + | Struct | Union | Enum | Typedef => TokenRole::Operator, // `CharLiteral` joins the operand list here for the whole // C family — `cpp.rs`, `mozcpp.rs` and `objc.rs` carry the // same arm and point back at this note (#1316). Before it a diff --git a/big-code-analysis-ast/src/getter/cpp.rs b/big-code-analysis-ast/src/getter/cpp.rs index 81f22d65..89ac98ee 100644 --- a/big-code-analysis-ast/src/getter/cpp.rs +++ b/big-code-analysis-ast/src/getter/cpp.rs @@ -4,6 +4,19 @@ use super::*; use crate::c_declarator::declarator_name; +// `GT2` is the template closer's own symbol, which the parser folds +// onto `GT` (`cpp_template_closer_alias_never_reaches_kind_id`); it is +// listed defensively. +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Cpp::TemplateArgumentList as u16, + Cpp::TemplateParameterList as u16, + ], + openers: &[Cpp::LT as u16], + closers: &[Cpp::GT as u16, Cpp::GT2 as u16], + is_misparse: never_misparsed, +}; + impl Getter for CppCode { fn get_func_space_name<'a, 'tree>( node: &Node<'tree>, @@ -63,8 +76,10 @@ impl Getter for CppCode { match node.kind_id().into() { FunctionDefinition | FunctionDefinition2 | FunctionDefinition3 | FunctionDefinition4 => SpaceKind::Function, - StructSpecifier => SpaceKind::Struct, - ClassSpecifier => SpaceKind::Class, + // Bodiless specifiers open no space (#1555); see + // `cpp_specifier_has_body`. + StructSpecifier if crate::checker::cpp_specifier_has_body(node) => SpaceKind::Struct, + ClassSpecifier if crate::checker::cpp_specifier_has_body(node) => SpaceKind::Class, NamespaceDefinition => SpaceKind::Namespace, TranslationUnit => SpaceKind::Unit, _ => SpaceKind::Unknown, @@ -74,6 +89,10 @@ impl Getter for CppCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Cpp::*; + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } + // `LPAREN2` here (and the `LBRACK2`/`LBRACK3` aliases in the // Elixir/Ruby impls) is a defensive arm, not an active one: every // grammar's `public_symbol_map` collapses the second-alias opener @@ -139,7 +158,52 @@ impl Getter for CppCode { // modifier has a distinct kind_id, so keying by kind_id (the default // `operators` store) keeps them distinct in n1 while `long long`'s // two `long` tokens correctly fold to one n1 entry but two N1 hits. - | Signed | Unsigned | Long | Short => TokenRole::Operator, + | Signed | Unsigned | Long | Short + // An overloaded operator's name (#1296). `operator +` already + // billed its `+` through the punctuation kind, but the grammar + // spells four overloadable names as whole tokens of their own + // — `[]`, `()`, the `""` of a user-defined literal and + // `co_await` — and none was in either arm, so + // `operator[]` and `operator()` billed nothing for the name + // that tells them apart. `Operator` is the `operator` keyword + // itself, which C# bills the same way. `CoAwait` is also the + // keyword of `co_await expr`, an operator there too. Each + // token occurs only under `operator_name`, `operator_cast` or + // `co_await_expression`, none of which is classified, so + // nothing bills one twice (grammar-dispatch section 5). + // + // `LPARENRPAREN` / `LBRACKRBRACK` are kinds of their own, not + // the `LPAREN` / `LBRACK` of a call or subscript, so declaring + // `operator[]` and applying `s[1]` are two `n1` entries, which + // `bca ops` renders `operator[]` and `[]` (see + // `get_operator_id_as_str`). + | LPARENRPAREN | LBRACKRBRACK | DQUOTEDQUOTE | CoAwait | Operator + // `co_return` and `co_yield` are the coroutine spellings of + // `return` and of handing a value back, so they bill as + // `return` does (#1547). Each is the keyword leaf of a + // `co_return_statement` / `co_yield_statement`, which no arm + // classifies, so nothing bills one twice. + | CoReturn | CoYield + // The type-declaration keywords, billed as Rust bills + // `struct` and C# its `class` / `struct` / `enum` (#1552). + // Each is the keyword leaf of a `class_specifier` / + // `struct_specifier` / `union_specifier` / `enum_specifier`, + // none of which is classified, so a specifier bills its + // keyword once — and `enum class` bills both of its two. + // The same leaf heads an elaborated type (`struct S s;`), + // where it is billed too, as `int` is in `int s;`. + // + // #1557 adds the remaining declaration keywords: `namespace`, + // `template`, `typename`, `typedef` and `using`, each the + // leaf of an unclassified wrapper (`namespace_definition`, + // `template_declaration`, `type_parameter_declaration`, + // `type_definition`, `alias_declaration` / + // `using_declaration`). A template-parameter `class` bills + // with them: it is the synonym of `typename`, so the two + // spellings of `template ` still score alike, now + // both billed. + | Class | Struct | Union | Enum | Namespace | Template | Typename | Typedef | Using + => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only // classified node in a character literal, so it bills one @@ -215,5 +279,28 @@ impl Getter for CppCode { } } - get_operator!(Cpp); + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + + // `get_operator!`'s three bracket arms, plus the two overloaded + // operator names (#1296). The grammar names `LPARENRPAREN` / + // `LBRACKRBRACK` `()` / `[]`, the same text a call's and a + // subscript's opener render as, so `bca ops` would list one spelling + // for two `n1` entries. Rendering each as the operator it names keeps + // every listed operator distinct (lesson 4). + fn get_operator_id_as_str(id: u16) -> &'static str { + let typ = id.into(); + match typ { + Cpp::LPAREN => "()", + Cpp::LBRACK => "[]", + Cpp::LBRACE => "{}", + Cpp::LPARENRPAREN => "operator()", + Cpp::LBRACKRBRACK => "operator[]", + _ => typ.into(), + } + } } diff --git a/big-code-analysis-ast/src/getter/csharp.rs b/big-code-analysis-ast/src/getter/csharp.rs index fa97fb89..00024916 100644 --- a/big-code-analysis-ast/src/getter/csharp.rs +++ b/big-code-analysis-ast/src/getter/csharp.rs @@ -3,6 +3,49 @@ use super::*; +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Csharp::TypeArgumentList as u16, + Csharp::TypeParameterList as u16, + ], + openers: &[Csharp::LT as u16], + closers: &[Csharp::GT as u16], + is_misparse: csharp_comparison_misparse, +}; + +// Whether `list` holds the two comparisons of a call argument +// (`F(a < b, c > d)`) that tree-sitter-c-sharp reads as a declaration +// typed `a` and named `d`. The C# specification keeps the type +// arguments only where the token after `>` allows it (§6.2.5), and an +// identifier allows it only after `is`, `case`, `out` or in a tuple — +// so a declaration typed by a generic name is two comparisons when it is +// a call or indexer argument without `out`. `F(out List x)`, +// `(List p, int q) = …` and `o is List x` keep their pair. A +// member access before the `<` (`F(x.a < b, c > d)`) wraps the generic +// name in `qualified_name`s, which the climb passes. The `>=` spelling +// (`F(a < b, c >= d)`) parses as an assignment instead and is not undone. +fn csharp_comparison_misparse<'a>(list: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let mut up = ancestors.iter(list).map(|(node, _)| node); + if up + .next() + .is_none_or(|name| name.kind_id() != Csharp::GenericName as u16) + { + return false; + } + let mut up = up.skip_while(|node| node.kind_id() == Csharp::QualifiedName as u16); + let (Some(declaration), Some(argument), Some(arguments)) = (up.next(), up.next(), up.next()) + else { + return false; + }; + declaration.kind_id() == Csharp::DeclarationExpression as u16 + && argument.kind_id() == Csharp::Argument as u16 + && !argument.is_child(Csharp::Out as u16) + && matches!( + arguments.kind_id().into(), + Csharp::ArgumentList | Csharp::BracketedArgumentList + ) +} + impl Getter for CsharpCode { fn get_space_kind(node: &Node) -> SpaceKind { use Csharp::*; @@ -48,6 +91,10 @@ impl Getter for CsharpCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Csharp::*; + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } + match node.kind_id().into() { // Control-flow keywords If | Else | Switch | Case | Default | Try | Catch | Finally | Throw @@ -85,14 +132,15 @@ impl Getter for CsharpCode { // the text-keyed `n2` hid it (#1253). The leaf cannot simply // be dropped: the overloadable-operator list emits a bare // `true` / `false` under `operator_declaration` — the - // grammar's only unwrapped position — and suppressing that - // would leave two such declarations with identical Halstead - // vocabularies whenever their bodies match. That position - // keeps its pre-#1253 `Operand` classification here; whether - // an overloaded operator's *name* is better counted as an - // operator, as `operator +` already is, is #1296. + // grammar's only unwrapped position. There the keyword + // *names* an operator, exactly as the `+` of `operator +` + // does, so it is billed as one (#1296); C# merely spells two + // of its overloadable operators with keywords instead of + // punctuation. The fallback keeps any other unwrapped + // position an operand, the role the keyword has as a value. True | False => match ancestors.parent(node).map(|p| p.kind_id().into()) { Some(BooleanLiteral) => TokenRole::Unknown, + Some(OperatorDeclaration) => TokenRole::Operator, _ => TokenRole::Operand, }, // `this` is a self-reference everywhere (`this.x`, @@ -137,7 +185,13 @@ impl Getter for CsharpCode { // spells inheritance (`class D : B`) holds the base type's // identifier and no `base` keyword, so nothing bills this // text twice. - Identifier | Base + // + // `ImplicitParameter` is the parameter of `x => x`: a + // childless leaf where `(x) => x` puts an `identifier` inside + // a `parameter_list`, so the bare spelling dropped its name + // from both Halstead halves (#1544). Nothing containing it + // is classified. + Identifier | Base | ImplicitParameter | IntegerLiteral | RealLiteral | BooleanLiteral | NullLiteral | CharacterLiteral | StringLiteral | VerbatimStringLiteral | RawStringLiteral => TokenRole::Operand, @@ -193,7 +247,7 @@ impl Getter for CsharpCode { // (grammar-dispatch section 7). // // `CsharpCode::is_primitive` routes the operator half through the - // lexeme-keyed map, because these five keywords now have two kind + // lexeme-keyed map, because these seven keywords have two kind // spellings for one operator; its comment carries that argument. fn get_op_type_with_code<'a>( node: &Node<'a>, @@ -205,21 +259,27 @@ impl Getter for CsharpCode { } match node_text(code, node) { Some("this") => TokenRole::Operand, - Some("ref" | "out" | "in" | "scoped" | "readonly") => TokenRole::Operator, - // Live, not merely defensive: `_lambda_expression_init` and - // `anonymous_method_expression` alias a bare `static` / - // `async` onto `modifier` too, so `static (int x) => x` - // arrives here. Leaving those unclassified is what they - // already were before #1418, and closing that gap is a - // decision about lambdas rather than about parameters — the - // *declaration* spelling of both keywords is billed through - // its leaf, so they are unbilled only in this one position. - // A grammar bump that adds a seventh parameter spelling - // lands here as well, unclassified rather than billed as - // whichever arm happened to be last. + // `_lambda_expression_init` and `anonymous_method_expression` + // alias a bare `static` / `async` onto `modifier` too, so + // `static (int x) => x` arrives here. The declaration spelling + // of both keywords is billed as an operator through its leaf, + // so the lambda spelling is billed the same way (#1482). + Some("ref" | "out" | "in" | "scoped" | "readonly" | "static" | "async") => { + TokenRole::Operator + } + // A grammar bump that adds another aliased spelling lands + // here unclassified rather than billed as whichever arm + // happened to be last. _ => TokenRole::Unknown, } } + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + get_operator!(Csharp); } diff --git a/big-code-analysis-ast/src/getter/elixir.rs b/big-code-analysis-ast/src/getter/elixir.rs index 19b1220f..c436f3a0 100644 --- a/big-code-analysis-ast/src/getter/elixir.rs +++ b/big-code-analysis-ast/src/getter/elixir.rs @@ -142,6 +142,17 @@ impl Getter for ElixirCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Elixir as E; + // A *named* operator (`&==/2`, `&and/2`, `Kernel.||(a, b)`) is one + // `operator_identifier` holding the operator token. The wrapper is + // the operand, as `foo` is in `&foo/2`, so the token inside it must + // not also bill an operator (grammar-dispatch §5, #1534). The + // wrapper is the keeper because it exists for every spelling: `..` + // is a childless `operator_identifier`, so keeping the leaf would + // lose it. + if ancestors.parent_has_kind(node, E::OperatorIdentifier as u16) { + return TokenRole::Unknown; + } + match node.kind_id().into() { // Sigil delimiter punctuation. The delimiter choice is // spelling, not semantics — `~r/abc/` is not two divisions diff --git a/big-code-analysis-ast/src/getter/generic_angles.rs b/big-code-analysis-ast/src/getter/generic_angles.rs new file mode 100644 index 00000000..ff67d09a --- /dev/null +++ b/big-code-analysis-ast/src/getter/generic_angles.rs @@ -0,0 +1,67 @@ +//! The generic angle brackets every grammar with generics shares with +//! its comparison operators (#1559). + +use crate::{Ancestors, Node}; + +/// How [`get_operator_spelling`] renders a generic angle-bracket pair, +/// in every language that has one (#1559). +/// +/// [`get_operator_spelling`]: crate::Getter::get_operator_spelling +pub(crate) const ANGLE_PAIR: &str = "<>"; + +/// The kinds one grammar spells its generic angle brackets with: the +/// list nodes whose `<` / `>` delimit type arguments or parameters +/// rather than compare, and the bracket tokens themselves. +/// +/// A list's brackets are one bracket pair, billed like `()`, `[]` and +/// `{}` (#1395): the opener carries the pair as [`ANGLE_PAIR`] and the +/// closer bills nothing (#1559). The grammars give the brackets the +/// same kind ids as the comparison operators, so only the parent tells +/// the two apart, and the opener is keyed by text rather than by kind +/// to keep it out of the comparison `<`'s vocabulary entry. +pub(crate) struct GenericAngleKinds { + /// The list nodes: type-argument, type-parameter and template + /// lists, plus each grammar's other spelling of a generic list. + pub(crate) lists: &'static [u16], + /// The `<` kinds, with any alias the grammar declares for it. + pub(crate) openers: &'static [u16], + /// The `>` kinds, likewise. + pub(crate) closers: &'static [u16], + /// Whether a list node, given its ancestor chain, holds comparisons + /// the grammar misparsed as a generic list, whose brackets then keep + /// billing as the comparisons they are. [`never_misparsed`] for a + /// grammar with no such shape this can tell from a real list. + pub(crate) is_misparse: for<'a, 'b> fn(&Node<'a>, Ancestors<'a, 'b>) -> bool, +} + +/// The [`GenericAngleKinds::is_misparse`] of a grammar whose generic +/// lists hold no misparse this can tell apart. C++ is one: tree-sitter-cpp +/// reads `a < b || c > (d)` as a template argument list, but so does a +/// valid template call spelled the same, and only name lookup tells them +/// apart. +pub(crate) fn never_misparsed(_: &Node, _: Ancestors) -> bool { + false +} + +impl GenericAngleKinds { + fn parent_is_list<'a>(&self, node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + ancestors.iter(node).next().is_some_and(|(list, above)| { + self.lists.contains(&list.kind_id()) && !(self.is_misparse)(&list, above) + }) + } + + /// Whether `node` closes a generic list, and so bills nothing. + pub(crate) fn is_closer<'a>(&self, node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + self.closers.contains(&node.kind_id()) && self.parent_is_list(node, ancestors) + } + + /// [`ANGLE_PAIR`] when `node` opens a generic list, else `None`. + pub(crate) fn opener_spelling<'a>( + &self, + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + (self.openers.contains(&node.kind_id()) && self.parent_is_list(node, ancestors)) + .then_some(ANGLE_PAIR) + } +} diff --git a/big-code-analysis-ast/src/getter/groovy.rs b/big-code-analysis-ast/src/getter/groovy.rs index 02aa49a5..a93d2a35 100644 --- a/big-code-analysis-ast/src/getter/groovy.rs +++ b/big-code-analysis-ast/src/getter/groovy.rs @@ -3,6 +3,23 @@ use super::*; +// `method_type_parameters` is the method-level spelling of +// `type_parameters` (` T id(T x)`). `LT2` is the `token.immediate` +// `<` that `type_parameters` and `type_arguments` open with; the parser +// reports it as `LT`, so it is listed defensively, here and in the +// operator arm, and its absence is pinned by +// `groovy_generic_opener_alias_never_reaches_kind_id`. +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Groovy::TypeArguments as u16, + Groovy::TypeParameters as u16, + Groovy::MethodTypeParameters as u16, + ], + openers: &[Groovy::LT as u16, Groovy::LT2 as u16], + closers: &[Groovy::GT as u16], + is_misparse: never_misparsed, +}; + impl Getter for GroovyCode { /// Names the space, synthesising one for constructs that carry no /// name token (#1184). @@ -82,12 +99,17 @@ impl Getter for GroovyCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Groovy::*; + + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } // Mirrors `JavaCode`'s minimal classification — modifiers - // (`Public`, `Static`, …), declaration keywords (`Class`, - // `Interface`, …), and module keywords (`Package`, `Import`, - // …) are excluded because they live inside `Modifiers` / - // `*Declaration` wrappers and would over-count if treated as - // separate operators. The dekobon Groovy grammar (#246, #247) + // (`Public`, `Static`, …) and module keywords (`Package`, + // `Import`, …) are excluded. The type-declaration keywords + // (`class`, `interface`, `trait`, `enum`, `record`) are billed, + // as Java's are since #1552: no `*Declaration` wrapper is + // classified, so the keyword leaf is the only node that can + // bill one. The dekobon Groovy grammar (#246, #247) // emits a distinct named node for every Groovy-specific // operator (Elvis `?:`, safe-nav `?.`, identity `===`/`!==`, // regex `=~`/`==~`, spaceship `<=>`, exclusive ranges @@ -156,16 +178,35 @@ impl Getter for GroovyCode { Super if ancestors.parent_has_kind(node, Wildcard as u16) => TokenRole::Operator, Super => TokenRole::Operand, + // `trait` and `record` are contextual keywords, so valid + // Groovy may use either as a name. Inside a class the grammar + // parses that name as an `identifier`, but at script level it + // fails `record = 3`, `String record` and `void record() {}` + // into an `ERROR` holding the keyword leaf. Billing the leaf + // there would make a variable an operator, so it bills only + // under the declaration it introduces (#1552). + Trait if ancestors.parent_has_kind(node, TraitDeclaration as u16) => { + TokenRole::Operator + } + Record if ancestors.parent_has_kind(node, RecordDeclaration as u16) => { + TokenRole::Operator + } + // Control-flow + keyword operators (mirrors Java's set, // minus tokens that no longer exist in the dekobon grammar // — `This`, `VoidType`, `Throws2`). If | Else | Switch | Case | Try | Catch | Throw | Throws | For | While | Continue | Break | Do | Finally | New | Return | Default | Abstract | Assert | Instanceof | Extends | Final | Implements | Transient | Synchronized | Def | In | As + // Type-declaration keywords (#1552). Unlike Java's, `class` + // needs no class-literal guard: `String.class` parses as a + // `field_access` whose `class` is a plain `identifier`. + // `trait` and `record` are the gated arms above. + | Class | Interface | Enum // Separators / brackets. | SEMI | COMMA | COLONCOLON | DOT | DASHGT | LBRACE | LBRACK | LPAREN // Java-compatible operators (arithmetic, bitwise, comparison, assignment). - | EQ | LT | GT | BANG | TILDE | QMARK | COLON | EQEQ | LTEQ | GTEQ | BANGEQ + | EQ | LT | LT2 | GT | BANG | TILDE | QMARK | COLON | EQEQ | LTEQ | GTEQ | BANGEQ | AMPAMP | PIPEPIPE | PLUSPLUS | DASHDASH | PLUS | DASH | STAR | SLASH | AMP | PIPE | CARET | PERCENT | LTLT | GTGT | GTGTGT | PLUSEQ | DASHEQ | STAREQ | SLASHEQ | AMPEQ | PIPEEQ | CARETEQ | PERCENTEQ | LTLTEQ | GTGTEQ | GTGTGTEQ @@ -228,6 +269,13 @@ impl Getter for GroovyCode { } } + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + fn get_operator_id_as_str(id: u16) -> &'static str { let typ = id.into(); match typ { diff --git a/big-code-analysis-ast/src/getter/java.rs b/big-code-analysis-ast/src/getter/java.rs index 20e35167..b5439162 100644 --- a/big-code-analysis-ast/src/getter/java.rs +++ b/big-code-analysis-ast/src/getter/java.rs @@ -3,6 +3,13 @@ use super::*; +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[Java::TypeArguments as u16, Java::TypeParameters as u16], + openers: &[Java::LT as u16], + closers: &[Java::GT as u16], + is_misparse: never_misparsed, +}; + impl Getter for JavaCode { /// Names the space, synthesising one for constructs that carry no /// name token (#1184). @@ -80,16 +87,45 @@ impl Getter for JavaCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Java::*; + + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } // Some guides that informed grammar choice for Halstead // keywords, operators, literals: https://docs.oracle.com/javase/specs/jls/se18/html/jls-3.html#jls-3.12 // https://www.geeksforgeeks.org/software-engineering-halsteads-software-metrics/?msclkid=5e181114abef11ecbb03527e95a34828 match node.kind_id().into() { + // `class` declares a class (billed below, #1552) except as the + // tail of a class literal (`C.class`): that `class_literal` is + // already one operand keyed by its whole text, so billing its + // leaf too would pay the same bytes twice (grammar-dispatch + // section 5, the compound-leaf guard). + Class if ancestors.parent_has_kind(node, ClassLiteral as u16) => TokenRole::Unknown, + // A type name is an operand (#1560, the operand arm below) + // except inside a class literal, whose whole text is already + // one operand — the `class` guard above, for the same + // reason. The type sits there directly (`Foo.class`) or + // under an array or scoped type (`Foo[].class`, + // `java.util.List.class`), so the climb skips those two. + TypeIdentifier + if ancestors + .iter(node) + .map(|(ancestor, _)| ancestor.kind_id()) + .find(|&id| id != ArrayType as u16 && id != ScopedTypeIdentifier as u16) + == Some(ClassLiteral as u16) => + { + TokenRole::Unknown + } // Operator: control flow | If | Else | Switch | Case | Try | Catch | Throw | Throws | Throws2 | For | While | Continue | Break | Do | Finally // Operator: keywords | New | Return | Default | Abstract | Assert | Instanceof | Extends | Final | Implements | Transient | Synchronized | VoidType + // Operator: type-declaration keywords, billed as C# bills + // its own (#1552). Only the leaves: no `*_declaration` + // wrapper is listed, so a declaration bills its keyword once. + | Class | Interface | Enum | Record | ATinterface // Operator: brackets and comma and terminators (separators) | SEMI | COMMA | COLONCOLON | DOT | DASHGT | LBRACE | LBRACK | LPAREN // Operator: operators @@ -125,8 +161,16 @@ impl Getter for JavaCode { // `csharp.rs`, deliberately: a parameter name is an operand // everywhere here, while a *member* named by a keyword sits // with `operator +`. - Identifier | NullLiteral | ClassLiteral | True | False | StringLiteral - | CharacterLiteral | HexIntegerLiteral | OctalIntegerLiteral + // + // `TypeIdentifier` joined in #1560, as C#, Rust, the C family, + // Go, Groovy and TypeScript bill type names: a field or + // parameter type, an `implements` / `extends` entry, a type + // parameter, a generic argument, each segment of + // `java.util.List` (the `.` bills as itself). It is the one + // type-name kind and no listed node wraps it, so each name + // bills once. + Identifier | TypeIdentifier | NullLiteral | ClassLiteral | True | False + | StringLiteral | CharacterLiteral | HexIntegerLiteral | OctalIntegerLiteral | BinaryIntegerLiteral | DecimalIntegerLiteral | HexFloatingPointLiteral | DecimalFloatingPointLiteral | This => { TokenRole::Operand @@ -137,6 +181,32 @@ impl Getter for JavaCode { } } + // `var x = …` parses `var` as a `type_identifier` (it is a reserved + // type name, not a keyword token), so the #1560 operand arm would + // bill an operand spelled `var` that names no type. It stays + // unbilled, as C#'s `var` (an `implicit_type` wrapping its own + // keyword) is; a variable *named* `var` is an `identifier` and is + // unaffected. Only the bytes tell the two type names apart, so the + // byte-less `get_op_type` above still answers `Operand` for it; the + // walk calls this spelling (grammar-dispatch section 7). + fn get_op_type_with_code<'a>( + node: &Node<'a>, + code: &[u8], + ancestors: Ancestors<'a, '_>, + ) -> TokenRole { + if node.kind_id() == Java::TypeIdentifier && node_text(code, node) == Some("var") { + return TokenRole::Unknown; + } + Self::get_op_type(node, ancestors) + } + + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + fn get_operator_id_as_str(id: u16) -> &'static str { let typ = id.into(); match typ { diff --git a/big-code-analysis-ast/src/getter/javascript.rs b/big-code-analysis-ast/src/getter/javascript.rs index fd3b7ea4..c6c1c787 100644 --- a/big-code-analysis-ast/src/getter/javascript.rs +++ b/big-code-analysis-ast/src/getter/javascript.rs @@ -54,10 +54,13 @@ impl Getter for JavascriptCode { // JSX closing / self-closing tag delimiters. See the macro's // `$op_extras` note: they are per-language extras because // TypeScript, which shares this macro, has no JSX (#1395). + // `jsx:` bills element text and its entities as operands, for the + // same reason (#1483). impl_js_family_get_op_type!( Javascript, op_extras: [OptionalChain, LTSLASH, SLASHGT], operand_extras: [Identifier2, String2], + jsx: [JsxText, HtmlCharacterReference], ); get_operator!(Javascript); diff --git a/big-code-analysis-ast/src/getter/js_family.rs b/big-code-analysis-ast/src/getter/js_family.rs new file mode 100644 index 00000000..dc9aea02 --- /dev/null +++ b/big-code-analysis-ast/src/getter/js_family.rs @@ -0,0 +1,331 @@ +//! The `get_op_type` template the four JS-family grammars share +//! (JavaScript, MozJS, TypeScript, TSX), kept apart from the trait so +//! the per-language deltas and their rationale read as one unit. + +// Emit a `Getter::get_op_type` body for a JS-family language. The four +// JS-family grammars (JavaScript, MozJS, TypeScript, TSX) share most of +// their Halstead operator/operand kind classifications; per-language +// deltas are passed as bracketed extras so all four impls stay in +// lockstep when a kind is added or removed (issue #299). +// +// `$op_extras` per language: +// * JavaScript / MozJS: `OptionalChain` — the bare `?.` token (these +// grammars expose no `optional_chain` wrapper). +// * TypeScript / TSX: `QMARKDOT`, `PredefinedType` — `QMARKDOT` is +// the bare `?.` token under the `optional_chain` wrapper (issue +// #281); `PredefinedType` is the TS type keyword set (`string`, +// `number`, `boolean`, …). +// * JavaScript / MozJS / TSX: `LTSLASH`, `SLASHGT` — the JSX closing +// and self-closing tag delimiters, which these grammars spell as +// their own two-character tokens rather than as a `<` / `>` pair. +// Until #1395 neither was in either arm, so a JSX element billed +// its opening `<` and its `>`s but nothing for `` +// reported one bracket operator where the source spells two. They +// are extras rather than members of the shared arm because +// TypeScript is the one grammar of the four with no JSX and +// therefore no such variants to name. +// +// `$operand_extras` per language: +// * JavaScript / MozJS / TSX: `Identifier2`, `String2` — anonymous +// keyword aliases the JS grammar exposes for `Identifier` and +// `String`. TSX's `String2` (kind_id 261) is its string-literal +// alias, so it stays an operand like JS's. +// * TypeScript: none. TS's own aliases are either operators or +// deliberately unclassified (below). +// The `: string` type-keyword aliases (TS `String2`, kind_id 135; +// TSX `String3`, kind_id 141) are deliberately absent: they are +// emitted only as the child of a `predefined_type` wrapper, which +// already contributes the text-keyed `"string"` operator, and +// #313's listing of the keyword child as an operand counted one +// source token twice (#1261, which also narrowed +// `Checker::is_string` to match). +// +// **A member access contributes its leaves, never the composite.** +// `a.b` is the operands `a` and `b` plus the `.` operator — three +// vocabulary entries, not four. Until #1263 the operand arm listed the +// `member_expression` wrapper *and* the walker descended into its +// `object` / `property` children, so every access billed one extra +// `N2` entry keyed on the whole `a.b` text (and `a.b.c` billed two: +// `a.b` and `a.b.c`). The composite is what the rest of the workspace +// already excludes — C, C++, Java, Rust, Python, Go, Kotlin, Ruby, Lua +// and PHP were all measured leaves-only for the identical shape — so +// identical code scored differently by language. The same call applies +// to TS's `nested_identifier` (`namespace N.M`), C#'s +// `qualified_name` / `generic_name` / `alias_qualified_name`, and +// Groovy's `qualified_name` / `qualified_type`. +// +// This is grammar-dispatch section 5's "any predicate listing both a +// container and a kind it can contain double-counts", and its section +// 6 corollary is why `PrivatePropertyIdentifier` joins the operand arm +// in the same change: `this.#x`'s `#x` leaf was in no operand list, so +// the composite had been its only count, and a bare deletion would +// have regressed private-field access to zero operands. (The `#x` in +// the *field definition* `#x = 1` had never been counted at all — no +// wrapper covered it — so this fixes that half too.) +// +// `jsx: [, ]` (JavaScript, MozJS, TSX) bills +// the content of a JSX element (#1483). A `jsx_text` node is an +// operand, the way a string's contents are; it is childless and no +// node containing it (`jsx_element`) is classified, so nothing bills it +// twice. Two refinements need the source bytes and so live in the +// `get_op_type_with_code` / `get_operand_id` overrides the parameter +// also emits: +// +// * A whitespace-only `jsx_text` is `Unknown`. The scanner emits no +// token for whitespace that starts with a newline, but same-line +// spaces between tags (` `) are a node of their own, and +// billing them would make `N2` follow the source's layout. +// * The operand is keyed on the text with its surrounding whitespace +// trimmed. A token keeps the newlines and indentation around it, so +// `Start` at two indentation depths would otherwise be two distinct +// operands. This is the same shape as Kotlin's #454 narrowing. +// Only the ends are trimmed: a multi-line text keeps its inner +// newlines and indentation, so the same lines at two depths, or +// wrapped differently, stay distinct operands although JSX joins +// them into one string. Folding them needs an owned key, and +// `get_operand_id` returns a slice of the source. +// +// An `html_character_reference` (`&`) splits the text around it +// into siblings, so `a & b` is three operands: `a`, `&` and +// `b`. Each node is one vocabulary entry, and the entity denotes a +// character the surrounding text does not, so it is not folded into +// either neighbour. The same kind also sits inside a JSX attribute's +// `string`, which is already the operand, so it is billed only when +// its parent is not a string. +// +// `Checker::is_string` deliberately does not list `jsx_text`: `find +// string` reports delimited string literals, the set the alterator +// flattens (#283), and element text is undelimited markup content. +// +// The `TemplateString` interpolation guard is shared verbatim (issue +// #192): a bare `` `...` `` mirrors a `"..."` operand, but an +// interpolated template must yield `Unknown` because its inner +// `TemplateSubstitution` expressions are walked separately. +// +// The `Regex` arms are shared the same way (issue #1314), and for the +// same reason: all four grammars spell a regex literal identically — +// a `regex` wrapper whose two delimiters are `SLASH`, the kind id real +// division uses — so neither arm has a per-language delta to hoist to +// a call site. Only the ids differ (`SLASH` 87/81/90/87, `Regex` +// 224/250/264/225) and the macro names both through the enum, so no +// invocation mentions either. See the arms themselves for the +// fabrication and the missing-operand halves. +macro_rules! impl_js_family_get_op_type { + ( + $lang:ident, + op_extras: [$($op_extra:ident),* $(,)?], + operand_extras: [$($operand_extra:ident),* $(,)?] + $(, predefined_void: $predefined_type:ident)? + $(, generic_angles: $generic_angles:ident)? + $(, export_type_name: [$type_keyword:ident, $export_clause:ident, $export_specifier:ident])? + $(, jsx: [$jsx_text:ident, $jsx_entity:ident])? $(,)? + ) => { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { + use $lang::*; + + $( + if $generic_angles.is_closer(node, ancestors) { + return TokenRole::Unknown; + } + )? + + // TS/TSX only: a binding *named* `type` in an export clause + // (`export { type }`, `export { type as kind }`). The grammar's + // `export_specifier` has no slot for that name, unlike + // `import_specifier`, so it recovers the keyword leaf under an + // `ERROR` in the clause, or beside one in the specifier, and + // the `type` operator would bill a binding name. A type-only + // specifier (`export { type Foo }`) parses cleanly and still + // bills, as does the `type` of `export type * from "m"`, whose + // `ERROR` hangs from the statement rather than the clause. + $( + if node.kind_id() == $type_keyword as u16 && { + let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); + up.next().is_some_and(|parent| { + if parent.is_error() { + up.next() + .is_some_and(|clause| clause.kind_id() == $export_clause as u16) + } else { + parent.kind_id() == $export_specifier as u16 && parent.has_error() + } + }) + } { + return TokenRole::Unknown; + } + )? + + // TS/TSX only: a `void` return / parameter type is parsed as a + // `predefined_type` wrapper around an inner `void` token. Both + // the wrapper (routed through `is_primitive` into the text-keyed + // `primitive_operators` map as `"void"`) and the inner `Void` + // token (a standalone expression operator, e.g. `void 0`) would + // otherwise classify as operators, double-counting one source + // `void` as two Halstead operators (issue #453). Only `void` has + // an operator-kind child; the `string` keyword's child is an + // operand-kind alias, which collided in the other direction + // (operator + operand) until #1261 dropped it from the operand + // extras. Suppress the wrapper here and + // let the inner `Void` token carry the single operator, keeping + // the kind_id-keyed count consistent with expression `void 0` + // (the lesson-4 `n1 == dedupe(ops.operators)` invariant). + $( + if node.kind_id() == $predefined_type as u16 + && node + .child(0) + .is_some_and(|child| child.kind_id() == Void as u16) + { + return TokenRole::Unknown; + } + )? + + match node.kind_id().into() { + // Regex delimiter punctuation. A `regex` literal spells + // both of its delimiters with the same kind id real + // division uses, so `const a = /abc/g;` reported a `/` + // operator with no division in the source and n1/N1 + // counted the literal's punctuation as arithmetic + // (#1314, the JS-family sibling of Elixir #1256 and + // Ruby/Perl #1312). The `Regex` node itself is the + // operand (below), so the delimiters are suppressed + // exactly when their parent is that node — the + // compound-leaf guard of grammar-dispatch section 5. + // + // Parent, not ancestor, for correctness by + // construction rather than by observation: unlike + // Ruby's `#{…}`, a JS regex admits no nested + // expression at all (`regex_pattern` and `regex_flags` + // are leaves), so no fixture can tell this guard from + // an ancestor-scanning one. The halstead test + // `js_regex_delimiter_guard_is_parent_scoped_is_unobservable` + // records that, and pins the grammar property it rests + // on, rather than implying coverage the suite lacks. + // + // `SLASH2` — the aliased regex-start token every one of + // these grammars carries — needs no arm: it is absent + // from the operator list below, so it already lands on + // `Unknown`, which is what a delimiter should be. That + // is the one place this differs from Ruby's guard, + // where `SLASH2` had to be moved off the arithmetic arm. + SLASH + if ancestors.parent_has_kind(node, Regex as u16) => + { + TokenRole::Unknown + } + Export | Import | Import2 | Extends | DOT | From | LPAREN | COMMA | As | STAR + | GTGT | GTGTGT | COLON | Return | Delete | Throw | Break | Continue | If + | Else | Switch | Case | Default | Async | Do | For | In | Of | While | Try + | Catch | Finally | With | EQ | AT | AMPAMP | PIPEPIPE | PLUS | DASH | DASHDASH + | PLUSPLUS | SLASH | PERCENT | STARSTAR | PIPE | AMP | LTLT | TILDE | LT | LTEQ + | EQEQ | BANGEQ | GTEQ | GT | PLUSEQ | BANG | BANGEQEQ | EQEQEQ | DASHEQ + | STAREQ | SLASHEQ | PERCENTEQ | STARSTAREQ | GTGTEQ | GTGTGTEQ | LTLTEQ | AMPEQ + | CARET | CARETEQ | PIPEEQ | Yield | LBRACK | LBRACE | Await | QMARK + | QMARKQMARK | EQGT | DOTDOTDOT | New | Let | Var | Const | Function + | SEMI | Typeof | Instanceof | Void + // `Function` and `Class2` are the `function` / `class` + // keyword leaves (#1552). Their expression wrappers — + // `FunctionExpression` and the unsuffixed `Class` — + // stay unlisted, so `const f = function () {}` bills + // one `function`, not two (#1554). The leaf is the + // keeper because it exists in every spelling: the + // `async` and generator forms, and an ERROR-recovery + // parse that drops the wrapper. + | Class2 + // `get`/`set` accessor keywords are operators, matching the + // C# getter's `Get | Set | Init | Add | Remove` accessor arm. + // Before #695 the JS family classified them as operands, so + // the same accessor keyword landed in opposite Halstead + // groups across languages, skewing n1/n2 for accessor-heavy + // code (#695). + | Set | Get + $(| $op_extra)* => TokenRole::Operator, + // `Regex` is the literal's own node and contributes one + // operand, the way Ruby's `Regex` and Elixir's `Sigil` + // do. It was in neither arm before #1314, so `/abc/g` + // reached the vocabulary from *neither* side: a + // fabricated `/` operator and no operand at all. Its + // `regex_pattern` / `regex_flags` children stay + // unclassified, so the wrapper cannot double-count + // them, and no interpolation is possible inside a JS + // regex — hence a plain arm here rather than the + // `string_operand_type` dispatch the template literal + // below needs. + // `PrivatePropertyIdentifier` is the `#x` leaf of a + // private class field, in both its declaration + // (`#x = 1`) and its access (`this.#x`). See the + // leaves-not-composites note above the macro for why + // it had to be added in the same change that dropped + // `MemberExpression*`. + // + // `MetaProperty` is `import.meta` / `new.target`: one + // atomic operand, like `this`, whose `meta` / `target` + // leaves are anonymous tokens no arm classifies. It is + // the one composite the #1263 drop has to keep + // (grammar-dispatch §6) — without it the meta-object + // contributes no operand at all while `this.env.x` + // still yields three. + Identifier | PropertyIdentifier | PrivatePropertyIdentifier | MetaProperty + | String | Number | True | False | Null | This | Super | Undefined | Regex + $(| $operand_extra)* => TokenRole::Operand, + // A `` `...` `` is a string literal; without interpolation it + // mirrors `"..."` and contributes one operand. When it has a + // `TemplateSubstitution` child the inner expression is already + // walked and classified, so counting the wrapper too would + // double-count its contribution to `N2` (issue #192, same + // pattern as #183 C# / #191 Kotlin / #199 Perl). + TemplateString => { + Self::string_operand_type(node, &[TemplateSubstitution as u16]) + } + $( + $jsx_text => TokenRole::Operand, + $jsx_entity + if !ancestors + .parent(node) + .is_some_and(|p| matches!(p.kind_id().into(), String | String2)) => + { + TokenRole::Operand + } + )? + _ => TokenRole::Unknown, + } + } + + $( + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + $generic_angles.opener_spelling(node, ancestors) + } + )? + + $( + fn get_op_type_with_code<'a>( + node: &Node<'a>, + code: &[u8], + ancestors: Ancestors<'a, '_>, + ) -> TokenRole { + if node.kind_id() == $lang::$jsx_text as u16 + && code[node.start_byte()..node.end_byte()].trim_ascii().is_empty() + { + return TokenRole::Unknown; + } + Self::get_op_type(node, ancestors) + } + + fn get_operand_id<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + ) -> &'a [u8] { + let text = &code[node.start_byte()..node.end_byte()]; + if node.kind_id() == $lang::$jsx_text as u16 { + text.trim_ascii() + } else { + text + } + } + )? + }; +} + +pub(super) use impl_js_family_get_op_type; diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index 7c2bad61..1255c0a1 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -122,6 +122,68 @@ fn kotlin_string_has_interp(node: &Node, code: &[u8]) -> bool { }) } +// Whether `node`, a modifier keyword leaf, sits in a modifier list the +// grammar built (#1558), rather than in the recovery of a soft-keyword +// *name* it failed to parse. Two recoveries strand the leaf: +// +// - an `ERROR` holds it, usually still wrapped in its `*_modifier` +// node (`open = !open` at statement start, `val suspend = 1`), so the +// first ancestor above the wrappers is the `ERROR`; +// - a modifier list is followed by an inserted, zero-width name +// (`fun f(vararg: Int)`, `class C(open: Int)`). +// +// Kinds are compared by name: `*_modifier` covers every wrapper, and +// `user_type` is where recovery can hang `type_modifiers`. The inserted +// name is found by width, not `is_missing()` (grammar-dispatch). +fn kotlin_modifier_was_written_as_one<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let host = ancestors + .iter(node) + .map(|(ancestor, _)| ancestor) + .find(|ancestor| { + let kind = ancestor.kind(); + !(kind.ends_with("_modifier") || kind == "type_modifiers" || kind == "user_type") + }); + host.is_some_and(|host| !host.is_error()) && !kotlin_next_token_is_inserted(node, ancestors) +} + +// Whether the first token after `node`, comments aside, is zero-width: +// one error recovery inserted where the source has none. A comment +// between the two is a sibling extra, so `vararg /*c*/: Int` must step +// over it to reach the inserted name, as `vararg: Int` does. A node +// never opens with an extra, which the parser hangs on the level above, +// so the descent takes each first child as it is. +// +// Every parameter's modifier asks, and the climb reaches +// `function_value_parameters`, so the sibling is found without a scan +// from the list's start: that made a long parameter list quadratic. +fn kotlin_next_token_is_inserted<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let mut current = *node; + ancestors + .iter(node) + .find_map(|(parent, _)| { + let next = parent + .children_after(¤t) + .find(|sibling| !sibling.as_tree_sitter().is_extra()); + current = parent; + next + }) + .is_some_and(|mut next| { + while let Some(first) = next.child(0) { + next = first; + } + next.start_byte() == next.end_byte() + }) +} + +// `super` (a `super_expression`) names a supertype rather than +// listing type arguments, so its brackets stay `<` and `>`. +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[Kotlin::TypeArguments as u16, Kotlin::TypeParameters as u16], + openers: &[Kotlin::LT as u16], + closers: &[Kotlin::GT as u16], + is_misparse: never_misparsed, +}; + impl Getter for KotlinCode { /// Names the space, synthesising one for constructs that carry no /// name token (#1184). @@ -210,16 +272,84 @@ impl Getter for KotlinCode { } } - fn get_op_type<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> TokenRole { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Kotlin::*; - match node.kind_id().into() { + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } + + // Modifier keywords, billed as C# bills all of its own (#1558; + // Java and Groovy bill only some, #1565). Each is the keyword + // leaf; its `*_modifier` / `modifiers` wrappers stay unlisted, so + // one modifier bills once (grammar-dispatch section 5). Every + // keyword has exactly one kind, so `kind_id` keys it without + // splitting it across the two `n1` maps. `Annotation2` is the + // `annotation` of `annotation class`; `Annotation` is an `@N` use, + // which is no keyword. `in` already bills as a variance modifier + // through `In` below, so `out` joins it. `reified` has only the + // `ReificationModifier` kind, which `KotlinCode::is_primitive` + // keys by its text. An early return rather than an arm, because + // the match below is one rustfmt declines to format. + // + // Each word is a soft keyword that valid Kotlin may use as a name, + // and where kotlin-ng fails to parse that name the leaf survives + // in its recovery (`open = !open` at statement start, + // `val suspend = 1`, a parameter named `vararg`). Such a leaf is + // no modifier and bills nothing; see + // `kotlin_modifier_was_written_as_one`. `companion` is left out + // of that gate: it never parses as a name, and gating it would + // only drop a real `companion object` from a class body that + // failed to parse. + let kind: Kotlin = node.kind_id().into(); + if matches!( + kind, + Enum | Sealed + | Annotation2 + | Data + | Inner + | Value + | Public + | Private + | Protected + | Internal + | Abstract + | Final + | Open + | Override + | Lateinit + | Const + | Tailrec + | Kotlin::Operator + | Infix + | Inline + | External + | Suspend + | Vararg + | Noinline + | Crossinline + | ReificationModifier + | Out + | Expect + | Actual + ) { + return if kotlin_modifier_was_written_as_one(node, ancestors) { + TokenRole::Operator + } else { + TokenRole::Unknown + }; + } + + match kind { // Operator: control flow keywords If | Else | When | For | While | Do | Try | Catch | Finally | Throw | Return | ReturnAT - // Operator: other keywords - | Class | Fun | Object | Val | Var | In | Is | As | AsQMARK | BANGis | BANGin - | Constructor + // Operator: other keywords. `interface` is the declaration + // keyword leaf beside `class` and `object` (#1556); its + // `class_declaration` wrapper stays unlisted. `companion` is + // the one modifier billed here; see the early return above. + | Class | Interface | Fun | Object | Val | Var | In | Is | As | AsQMARK | BANGis + | BANGin | Constructor | Companion // Operator: brackets, separators, terminators | SEMI | COMMA | COLONCOLON | DOT | LBRACE | LBRACK | LPAREN // Operator: assignment and arithmetic @@ -321,5 +451,12 @@ impl Getter for KotlinCode { } } + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + get_operator!(Kotlin); } diff --git a/big-code-analysis-ast/src/getter/mozcpp.rs b/big-code-analysis-ast/src/getter/mozcpp.rs index 450366b4..c822ec32 100644 --- a/big-code-analysis-ast/src/getter/mozcpp.rs +++ b/big-code-analysis-ast/src/getter/mozcpp.rs @@ -4,6 +4,17 @@ use super::*; use crate::c_declarator::declarator_name; +// `GT2` is listed defensively, as in the Cpp getter. +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Mozcpp::TemplateArgumentList as u16, + Mozcpp::TemplateParameterList as u16, + ], + openers: &[Mozcpp::LT as u16], + closers: &[Mozcpp::GT as u16, Mozcpp::GT2 as u16], + is_misparse: never_misparsed, +}; + impl Getter for MozcppCode { fn get_func_space_name<'a, 'tree>( node: &Node<'tree>, @@ -63,8 +74,10 @@ impl Getter for MozcppCode { match node.kind_id().into() { FunctionDefinition | FunctionDefinition2 | FunctionDefinition3 | FunctionDefinition4 => SpaceKind::Function, - StructSpecifier => SpaceKind::Struct, - ClassSpecifier => SpaceKind::Class, + // Bodiless specifiers open no space (#1555); see + // `cpp_specifier_has_body`. + StructSpecifier if crate::checker::cpp_specifier_has_body(node) => SpaceKind::Struct, + ClassSpecifier if crate::checker::cpp_specifier_has_body(node) => SpaceKind::Class, NamespaceDefinition => SpaceKind::Namespace, TranslationUnit => SpaceKind::Unit, _ => SpaceKind::Unknown, @@ -74,6 +87,10 @@ impl Getter for MozcppCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Mozcpp::*; + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } + // `LPAREN2` is a defensive arm (collapsed to `LPAREN` before // `kind_id()`; #768, see the Cpp note). match node.kind_id().into() { @@ -109,7 +126,17 @@ impl Getter for MozcppCode { // modifier has a distinct kind_id, so keying by kind_id (the default // `operators` store) keeps them distinct in n1 while `long long`'s // two `long` tokens correctly fold to one n1 entry but two N1 hits. - | Signed | Unsigned | Long | Short => TokenRole::Operator, + | Signed | Unsigned | Long | Short + // An overloaded operator's name, the `operator` keyword and + // `co_await` — the derivation is on the Cpp twin (#1296). + | LPARENRPAREN | LBRACKRBRACK | DQUOTEDQUOTE | CoAwait | Operator + // `co_return` / `co_yield`, billed as `return` is — the + // derivation is on the Cpp twin (#1547). + | CoReturn | CoYield + // The type-declaration keywords — the derivation is on the + // Cpp twin (#1552, #1557). + | Class | Struct | Union | Enum | Namespace | Template | Typename | Typedef | Using + => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only // classified node in a character literal, so it bills one @@ -142,5 +169,24 @@ impl Getter for MozcppCode { } } - get_operator!(Mozcpp); + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + + // The overloaded operator names render as the operator they name; + // the derivation is on the Cpp twin (#1296). + fn get_operator_id_as_str(id: u16) -> &'static str { + let typ = id.into(); + match typ { + Mozcpp::LPAREN => "()", + Mozcpp::LBRACK => "[]", + Mozcpp::LBRACE => "{}", + Mozcpp::LPARENRPAREN => "operator()", + Mozcpp::LBRACKRBRACK => "operator[]", + _ => typ.into(), + } + } } diff --git a/big-code-analysis-ast/src/getter/mozjs.rs b/big-code-analysis-ast/src/getter/mozjs.rs index 8fa2f02b..9486d448 100644 --- a/big-code-analysis-ast/src/getter/mozjs.rs +++ b/big-code-analysis-ast/src/getter/mozjs.rs @@ -54,10 +54,13 @@ impl Getter for MozjsCode { // JSX closing / self-closing tag delimiters. See the macro's // `$op_extras` note: they are per-language extras because // TypeScript, which shares this macro, has no JSX (#1395). + // `jsx:` bills element text and its entities as operands, for the + // same reason (#1483). impl_js_family_get_op_type!( Mozjs, op_extras: [OptionalChain, LTSLASH, SLASHGT], operand_extras: [Identifier2, String2], + jsx: [JsxText, HtmlCharacterReference], ); get_operator!(Mozjs); diff --git a/big-code-analysis-ast/src/getter/objc.rs b/big-code-analysis-ast/src/getter/objc.rs index 902ea448..c94315d4 100644 --- a/big-code-analysis-ast/src/getter/objc.rs +++ b/big-code-analysis-ast/src/getter/objc.rs @@ -4,6 +4,42 @@ use super::*; use crate::c_declarator::declarator_name; +// ObjC has no templates, but four nodes delimit a generic list with a +// bare `<` / `>`: `generic_specifier` (`NSArray`), +// `protocol_reference_list` (`id`), `parameterized_arguments` +// (`@interface A : NSObject `), and `argument_list`, whose +// `Type<…>` alternative (`f(NSArray)`) holds the brackets as +// direct children. A well-formed comparison's `<` / `>` is the child of +// a `binary_expression` or `preproc_binary_expression`; the one +// exception is `objc_argument_list_misparse`. `LT2` is the +// `token.immediate` `<` that `argument_list`'s form opens with; the +// parser reports it as `LT`, so it is listed defensively, here and in +// the operator arm, and its absence is pinned by +// `objc_generic_opener_alias_never_reaches_kind_id`. The call's +// `argument_list` is `ArgumentList2`; `ArgumentList` is +// `preproc_call_expression`'s alias, which holds no angle brackets. +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Objc::GenericSpecifier as u16, + Objc::ProtocolReferenceList as u16, + Objc::ParameterizedArguments as u16, + Objc::ArgumentList2 as u16, + ], + openers: &[Objc::LT as u16, Objc::LT2 as u16], + closers: &[Objc::GT as u16], + is_misparse: objc_argument_list_misparse, +}; + +// Whether `list` is an `argument_list` holding two comparisons that +// tree-sitter-objc read as its `Type<…>` form: an unspaced +// `g(ad)`, where a `<` glued to the first argument opens the form. +// The real form is the whole argument list, so `f(NSArray)` +// parses cleanly, while the misparse leaves an `ERROR` after the `>`, or +// a MISSING `)` (`g(i=0)`) that only `has_error` sees. +fn objc_argument_list_misparse(list: &Node, _ancestors: Ancestors) -> bool { + list.kind_id() == Objc::ArgumentList2 as u16 && list.has_error() +} + impl Getter for ObjcCode { fn get_func_space_name<'a, 'tree>( node: &Node<'tree>, @@ -70,6 +106,10 @@ impl Getter for ObjcCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Objc::*; + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } + // ObjC is C plus message sends, blocks, and the `@`-directives. // The operator alphabet is therefore the C set (`src/getter.rs` // `impl Getter for CCode`) extended with the ObjC structural @@ -93,14 +133,26 @@ impl Getter for ObjcCode { AT if ancestors.parent_has_kind(node, StringLiteral as u16) => TokenRole::Unknown, // The C operator set, then the ObjC-specific structural // keywords / markers from `In` onwards. + // + // From `Struct` onwards come the type-declaration keywords + // (#1552): C's `struct` / `union` / `enum` and, since #1557, + // `typedef` (see `c.rs`), and ObjC's own class and protocol + // declarations. Each is the leaf of a `type_definition` / + // `class_interface` / `class_implementation` / + // `protocol_declaration`, none of which is classified, so a + // declaration bills its keyword once. `@class Fwd;` needs no + // arm: it parses as an `@` token, already billed here, and a + // separate `class` leaf, so the directive already bills one + // operator. DOT | LPAREN | LPAREN2 | COMMA | STAR | GTGT | COLON | SEMI | Return | Break | Continue | If | Else | Switch | Case | Default | For | While | Goto | Do | EQ | AMPAMP | PIPEPIPE | DASH | DASHDASH | DASHGT | PLUS | PLUSPLUS | SLASH | PERCENT - | PIPE | AMP | LTLT | TILDE | LT | LTEQ | EQEQ | BANGEQ | GTEQ | GT | PLUSEQ + | PIPE | AMP | LTLT | TILDE | LT | LT2 | LTEQ | EQEQ | BANGEQ | GTEQ | GT | PLUSEQ | DASHEQ | BANG | STAREQ | SLASHEQ | PERCENTEQ | GTGTEQ | LTLTEQ | AMPEQ | CARET | CARETEQ | PIPEEQ | LBRACK | LBRACE | QMARK | PrimitiveType | TypeSpecifier | Sizeof | Signed | Unsigned | Long | Short | In | AT | ATtry | ATcatch | ATfinally - | ATthrow | ATsynchronized | ATautoreleasepool | ATselector | ATencode => { + | ATthrow | ATsynchronized | ATautoreleasepool | ATselector | ATencode | Struct + | Union | Enum | Typedef | ATinterface | ATimplementation | ATprotocol => { TokenRole::Operator } // `CharLiteral` — the full derivation lives on the same arm @@ -114,5 +166,12 @@ impl Getter for ObjcCode { } } + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + get_operator!(Objc); } diff --git a/big-code-analysis-ast/src/getter/perl.rs b/big-code-analysis-ast/src/getter/perl.rs index e2d706ea..12228f27 100644 --- a/big-code-analysis-ast/src/getter/perl.rs +++ b/big-code-analysis-ast/src/getter/perl.rs @@ -2,6 +2,10 @@ #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] use super::*; +use crate::lang_helpers::perl::{ + perl_and_is_operator, perl_dash_signs_keyword_key, perl_file_test_is_dash_key, + perl_is_dash_key_word, perl_key_is_signed, perl_not_is_key, +}; impl Getter for PerlCode { fn get_space_kind(node: &Node) -> SpaceKind { @@ -17,6 +21,9 @@ impl Getter for PerlCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Perl as P; + if let Some(role) = auto_quoted_key_role(node, ancestors) { + return role; + } match node.kind_id().into() { // Regex delimiter punctuation. The bare match form is the // only one of Perl's five regex literals that spells its @@ -37,6 +44,19 @@ impl Getter for PerlCode { { TokenRole::Unknown } + // The bare form's pattern text is one operand, billed on the + // `PatternMatcher` wrapper, but the grammar spells four of its + // characters as anonymous tokens under `regex_pattern_content`: + // `[`, `]`, `\` and `#`. They are literal regex text, which + // `m//` and `qr//` emit no node for, so `/[a]/` billed a `[]` + // operator that `m/[a]/` does not (#1549). `#` is also in no + // operator arm, because its only other position is a comment's + // leading `#` under `comments`, and a comment is not code. + // `$#a` and the `#` delimiters of `qw#…#` / `s#a#b#` are + // distinct token kinds. + _ if ancestors.parent_has_kind(node, P::RegexPatternContent as u16) => { + TokenRole::Unknown + } // Control-flow and declaration keywords. `Perl::Sub` is the // `sub` keyword (token id 16); `Perl::SUB` is the `__SUB__` // literal (token id 7) — that one is an operand, not an @@ -50,7 +70,7 @@ impl Getter for PerlCode { // Punctuation acting as operators | P::SEMI | P::COMMA | P::COLON | P::COLONCOLON | P::LBRACE | P::LBRACK | P::LPAREN | P::DOT | P::DOTDOT | P::DOTDOTDOT | P::FatComma | P::DASHGT - | P::QMARK | P::BSLASH | P::DOLLAR | P::DOLLARHASH | P::AT | P::PERCENT | P::HASH + | P::QMARK | P::BSLASH | P::DOLLAR | P::DOLLARHASH | P::AT | P::PERCENT // Arithmetic / comparison / logical / bitwise / assignment operators | P::EQ | P::PLUS | P::DASH | P::STAR | P::SLASH | P::STARSTAR | P::BANG | P::TILDE | P::EQTILDE | P::BANGTILDE | P::EQEQ | P::BANGEQ | P::LT | P::GT @@ -230,6 +250,34 @@ impl Getter for PerlCode { } } + /// Spells a `-bareword` key whole (#1545). A key that is exactly a + /// file test (`(-x => 1)`) has swallowed the `=>` and the value into + /// its node, so the key is the sign and the word after it; a + /// `-and` / `-not` key gets back the sign its `-` token stopped + /// billing, spelled the same however the two are spaced, since Perl + /// keys `(- not => 1)` as `"-not"` too. The sign is read off the tree + /// (`perl_key_is_signed`), not the byte before the key, which can + /// equally be an infix `-` or the end of a comment. + fn get_operand_id<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, + ) -> &'a [u8] { + let (start, end) = (node.start_byte(), node.end_byte()); + match node.kind_id().into() { + Perl::FileHandleOperator => { + let word = code[start + 1..end] + .iter() + .take_while(|byte| byte.is_ascii_alphanumeric() || **byte == b'_') + .count(); + &code[start..=start + word] + } + Perl::And if perl_key_is_signed(node, ancestors) => b"-and", + Perl::Not if perl_key_is_signed(node, ancestors) => b"-not", + _ => &code[start..end], + } + } + /// Folds paired delimiters to one glyph, and names the two /// pattern *operations* after their source spelling. /// @@ -240,7 +288,8 @@ impl Getter for PerlCode { /// `substitution_pattern_s` and `transliteration_tr_or_y` among /// the operators, which reads as a bug rather than as `s///`. /// `tr///` covers `y///` too — one kind, one glyph, because they - /// are synonyms. + /// are synonyms. `=>` is the named `fat_comma` node for the same + /// reason. #[inline] fn get_operator_id_as_str(id: u16) -> &'static str { let typ = id.into(); @@ -250,7 +299,68 @@ impl Getter for PerlCode { Perl::LBRACE => "{}", Perl::SubstitutionPatternS => "s///", Perl::TransliterationTrOrY => "tr///", + Perl::FatComma => "=>", _ => typ.into(), } } } + +/// The role of a token tree-sitter-perl 1.1.2 misparses inside an +/// auto-quoted hash key, or `None` for any other token. Perl quotes a +/// bareword before `=>` and alone in a hash subscript, keywords and a +/// leading `-` included, and the grammar reads some of those keys as +/// operators. Every arm is an upstream workaround; drop each with its +/// issue once the pin carries the fix. +fn auto_quoted_key_role<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> Option { + use Perl as P; + + let role = match node.kind_id().into() { + // FIXME(#1539 upstream): an auto-quoted `and` key (`$h{and}`, + // `(and => 1)`) lexes as the operator keyword. It is a + // string key, so it bills what `$h{or}` bills: the + // `identifier` operand the grammar gives every other + // bareword key. + P::And + if !ancestors + .parent(node) + .is_some_and(|parent| perl_and_is_operator(node, &parent)) => + { + TokenRole::Operand + } + // FIXME(#1541 upstream): the same for an auto-quoted `not` + // key (`$h{not}`, `(not => 1)`); see `perl_not_is_key`. + P::Not + if ancestors + .parent(node) + .is_some_and(|parent| perl_not_is_key(node, &parent)) => + { + TokenRole::Operand + } + // FIXME(#1545 upstream): a `-bareword` key (`$h{-foo}`, + // `(-text => 1)`) lexes as a file test on the rest of the + // word. It is one string key, so the test node bills it as + // one operand spelling the whole key, and the word inside + // bills nothing. A `-and` / `-not` key splits instead into + // a `-` and the keyword, which the arms above bill as the + // key; the `-` is part of it and bills nothing. See + // `perl_file_test_is_dash_key`. + P::FileHandleOperator if perl_file_test_is_dash_key(node, ancestors) => TokenRole::Operand, + P::Identifier + if ancestors + .iter(node) + .next() + .is_some_and(|(word, above)| perl_is_dash_key_word(&word, above)) => + { + TokenRole::Unknown + } + P::DASH + if ancestors + .parent(node) + .is_some_and(|parent| perl_dash_signs_keyword_key(node, &parent)) => + { + TokenRole::Unknown + } + _ => return None, + }; + Some(role) +} diff --git a/big-code-analysis-ast/src/getter/python.rs b/big-code-analysis-ast/src/getter/python.rs index f038eaad..1396941e 100644 --- a/big-code-analysis-ast/src/getter/python.rs +++ b/big-code-analysis-ast/src/getter/python.rs @@ -3,6 +3,30 @@ use super::*; +/// The role of a `{`, `:` or `=`, which Python spells both as syntax +/// and inside an f-string's replacement field. +/// +/// As syntax each is an operator (#1486). The interpolation opener +/// (`f"{x}"`, and a nested `{w}` inside a format spec) is spelling, not +/// an operation — the six interpolating languages settled that in #1314 +/// and Python follows them. The `:` that opens a format spec +/// (`f"{x:>10}"`) introduces the format mini-language, whose text is +/// string content and counts as nothing, so its delimiter does not +/// count either. The self-documenting `=` (`f"{x=}"`) is a directive of +/// the same field: it asks for the expression's text to be printed and +/// assigns nothing, like the `!r` conversion beside it, which was +/// already unbilled (#1551). +fn replacement_field_role(kind: Python, parent: Option) -> TokenRole { + use Python::*; + + match (kind, parent) { + (LBRACE | EQ, Some(Interpolation | FormatExpression)) | (COLON, Some(FormatSpecifier)) => { + TokenRole::Unknown + } + _ => TokenRole::Operator, + } +} + impl Getter for PythonCode { fn get_space_kind(node: &Node) -> SpaceKind { match node.kind_id().into() { @@ -16,7 +40,18 @@ impl Getter for PythonCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Python::*; - match node.kind_id().into() { + let kind: Python = node.kind_id().into(); + // Bracket openers, `;`, `:` and `=` are vocabulary like every + // other getter's (#1395, #1486); only the f-string + // replacement-field spellings are withdrawn. An early return + // rather than an arm, because the match below is one rustfmt + // declines to format. + if matches!(kind, LBRACE | COLON | EQ) { + let parent = ancestors.parent(node).map(|p| p.kind_id().into()); + return replacement_field_role(kind, parent); + } + + match kind { // The `not` / `in` / `is` leaf tokens are operators on their own // (`not x`, `a in b`, `a is b`, `for x in y`), but the grammar // also nests them inside the compound `not in` (Notin) and @@ -30,11 +65,12 @@ impl Getter for PythonCode { }, Import | DOT | From | COMMA | As | STAR | GTGT | Assert | COLONEQ | Return | Def | Del | Raise | Pass | Break | Continue | If | Elif | Else | Async | For - | While | Try | Except | Finally | With | DASHGT | EQ | Global | Nonlocal | Exec + | While | Try | Except | Finally | With | DASHGT | Global | Nonlocal | Exec | AT | And | Or | PLUS | DASH | SLASH | PERCENT | SLASHSLASH | STARSTAR | PIPE | AMP | CARET | LTLT | TILDE | LT | LTEQ | EQEQ | BANGEQ | GTEQ | GT | LTGT | PLUSEQ | DASHEQ | STAREQ | SLASHEQ | ATEQ | SLASHSLASHEQ | PERCENTEQ | STARSTAREQ - | GTGTEQ | LTLTEQ | AMPEQ | CARETEQ | PIPEEQ | Yield | Print + | GTGTEQ | LTLTEQ | AMPEQ | CARETEQ | PIPEEQ | Yield | Print | LPAREN | LBRACK + | SEMI // `not in` / `is not` compounds count as one operator each; the // inner Not/In/Is leaves are suppressed by the parent-guard arm // above (#413). @@ -53,7 +89,11 @@ impl Getter for PythonCode { // `lambda`: count only the keyword token (Lambda3=73), not the // Lambda/Lambda2 expression nodes that wrap it, to avoid the same // node+keyword double count fixed for await (#413). - | Lambda3 => { + | Lambda3 + // `class`, the declaration keyword beside `def`. Only the leaf: + // its `ClassDefinition` wrapper is not listed, so the pair + // cannot double-count (#1550). + | Class => { TokenRole::Operator } Identifier | Integer | Float | True | False | None => TokenRole::Operand, @@ -92,7 +132,5 @@ impl Getter for PythonCode { } } - fn get_operator_id_as_str(id: u16) -> &'static str { - Into::::into(id).into() - } + get_operator!(Python); } diff --git a/big-code-analysis-ast/src/getter/rust.rs b/big-code-analysis-ast/src/getter/rust.rs index bce73cd0..be49e11f 100644 --- a/big-code-analysis-ast/src/getter/rust.rs +++ b/big-code-analysis-ast/src/getter/rust.rs @@ -3,6 +3,24 @@ use super::*; +// `for_lifetimes` (`for<'a> fn(&'a u8)`) and `use_bounds` +// (`impl Tr + use<'a, T>`) are generic-parameter lists under other +// names. `LT2` is the `token(prec(1, '<'))` spelling `type_arguments` +// and `use_bounds` open with; the parser reports it as `LT`, so it is +// listed defensively, here and in the operator arm, and its absence is +// pinned by `rust_generic_opener_alias_never_reaches_kind_id`. +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Rust::TypeArguments as u16, + Rust::TypeParameters as u16, + Rust::ForLifetimes as u16, + Rust::UseBounds as u16, + ], + openers: &[Rust::LT as u16, Rust::LT2 as u16], + closers: &[Rust::GT as u16], + is_misparse: never_misparsed, +}; + impl Getter for RustCode { fn get_func_space_name<'a, 'tree>( node: &Node<'tree>, @@ -36,6 +54,9 @@ impl Getter for RustCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Rust::*; + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } match node.kind_id().into() { // `||` is treated as an operator only if it's part of a binary expression. // This prevents misclassification inside macros where closures without arguments (e.g., `let closure = || { /* ... */ };`) @@ -69,16 +90,16 @@ impl Getter for RustCode { // statement-heavy code. LPAREN | LBRACE | LBRACK | As | EQGT | PLUS | STAR | Async | Await | Break | Continue | Else | For | If | In | Let | Loop | Match | Return | Unsafe | While - | EQ | COMMA | DASHGT | QMARK | LT | GT | AMP | MutableSpecifier | DOTDOT + | EQ | COMMA | DASHGT | QMARK | LT | LT2 | GT | AMP | MutableSpecifier | DOTDOT | DOTDOTEQ | DASH | AMPAMP | PIPE | CARET | EQEQ | BANGEQ | LTEQ | GTEQ | LTLT | GTGT | PERCENT | PLUSEQ | DASHEQ | STAREQ | SLASHEQ | PERCENTEQ | AMPEQ | PIPEEQ | CARETEQ | LTLTEQ | GTGTEQ | Move | DOT | PrimitiveType | PrimitiveType2 | PrimitiveType3 | PrimitiveType4 | PrimitiveType5 | PrimitiveType6 | PrimitiveType7 | PrimitiveType8 | PrimitiveType9 | PrimitiveType10 | PrimitiveType11 | PrimitiveType12 | PrimitiveType13 | PrimitiveType14 - | PrimitiveType15 | PrimitiveType16 | PrimitiveType17 | Fn | SEMI | COLONCOLON - | Const | Static | Enum | Struct | Trait | Impl | Use | Mod | Pub | Type | Union - | Where | Extern | Dyn => TokenRole::Operator, + | PrimitiveType15 | PrimitiveType16 | PrimitiveType17 | Fn | SEMI | COLON + | COLONCOLON | Const | Static | Enum | Struct | Trait | Impl | Use | Mod | Pub + | Type | Union | Where | Extern | Dyn => TokenRole::Operator, // FieldIdentifier (e.g. `p.x`) and TypeIdentifier (e.g. `Vec`, // `HashMap`) are operand-class names — C++ and Go classify them // the same way (see arms ~588 and ~862 below). Omitting them @@ -93,5 +114,12 @@ impl Getter for RustCode { } } + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + get_operator!(Rust); } diff --git a/big-code-analysis-ast/src/getter/tsx.rs b/big-code-analysis-ast/src/getter/tsx.rs index 6eaf084f..d71365b8 100644 --- a/big-code-analysis-ast/src/getter/tsx.rs +++ b/big-code-analysis-ast/src/getter/tsx.rs @@ -3,6 +3,13 @@ use super::*; +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[Tsx::TypeArguments as u16, Tsx::TypeParameters as u16], + openers: &[Tsx::LT as u16], + closers: &[Tsx::GT as u16], + is_misparse: never_misparsed, +}; + impl Getter for TsxCode { fn get_space_kind(node: &Node) -> SpaceKind { use Tsx::*; @@ -73,12 +80,33 @@ impl Getter for TsxCode { // `LTSLASH` / `SLASHGT` are the JSX closing and self-closing tag // delimiters. TSX is the only one of the two TypeScript grammars // with JSX, so the `.ts` invocation has no such variants to pass - // (#1395). + // (#1395), and the same holds for the `jsx:` text kinds (#1483). + // + // `Interface` / `Enum` are the TS-only type-declaration keyword + // leaves, billed as the shared `class` is (#1552) and as C# and + // Java bill theirs; their `*_declaration` wrappers stay unlisted. + // + // `Type` / `Namespace` / `Module2` / `Declare` / `Global` and the + // `TypeIdentifier` operand: see the TS invocation (#1557), as for + // `export_type_name`. + // + // `Satisfies` / `Keyof` / `Infer` / `Is` / `Asserts2`, the type-level + // operator keywords: see the TS invocation (#1561). + // + // `generic_angles`: see the TS invocation (#1559). A JSX tag's `<` / + // `>` hang from the JSX element nodes, not from a generic list, so + // they stay `<` and `>`. impl_js_family_get_op_type!( Tsx, - op_extras: [QMARKDOT, PredefinedType, LTSLASH, SLASHGT], - operand_extras: [Identifier2, String2], + op_extras: [ + QMARKDOT, PredefinedType, Interface, Enum, Type, Namespace, Module2, Declare, Global, + Satisfies, Keyof, Infer, Is, Asserts2, LTSLASH, SLASHGT, + ], + operand_extras: [Identifier2, String2, TypeIdentifier], predefined_void: PredefinedType, + generic_angles: GENERIC_ANGLES, + export_type_name: [Type, ExportClause, ExportSpecifier], + jsx: [JsxText, HtmlCharacterReference], ); get_operator!(Tsx); diff --git a/big-code-analysis-ast/src/getter/typescript.rs b/big-code-analysis-ast/src/getter/typescript.rs index a082d555..63e458ab 100644 --- a/big-code-analysis-ast/src/getter/typescript.rs +++ b/big-code-analysis-ast/src/getter/typescript.rs @@ -3,6 +3,16 @@ use super::*; +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Typescript::TypeArguments as u16, + Typescript::TypeParameters as u16, + ], + openers: &[Typescript::LT as u16], + closers: &[Typescript::GT as u16], + is_misparse: never_misparsed, +}; + impl Getter for TypescriptCode { fn get_space_kind(node: &Node) -> SpaceKind { use Typescript::*; @@ -52,15 +62,15 @@ impl Getter for TypescriptCode { bound_name.map_or(Some(""), |name| node_text(code, &name)) } - // TypeScript's operand extras are empty. `NestedIdentifier` and - // `MemberExpression4` — the TS-only member-expression productions — - // were listed until #1263 and are now deliberately absent, matching - // the `MemberExpression*` drop in the macro body: `namespace N.M` - // contributes the operands `N` and `M` plus the `.` operator, and - // `a.b` contributes `a` and `b`, never the composite text as well. - // The composite was billed on top of leaves the walker already - // reached, which is grammar-dispatch section 5's container/leaf - // double-count. + // TypeScript's only operand extra is `TypeIdentifier` (#1557, + // below). `NestedIdentifier` and `MemberExpression4` — the TS-only + // member-expression productions — were listed until #1263 and are + // now deliberately absent, matching the `MemberExpression*` drop in + // the macro body: `namespace N.M` contributes the operands `N` and + // `M` plus the `.` operator, and `a.b` contributes `a` and `b`, + // never the composite text as well. The composite was billed on top + // of leaves the walker already reached, which is grammar-dispatch + // section 5's container/leaf double-count. // // TS's anonymous `"string"` alias `String2` (kind_id 135, the // `: string` type keyword, emitted only as the child of a @@ -72,11 +82,65 @@ impl Getter for TypescriptCode { // once (#1261). `Checker::is_string` no longer matches the keyword // either, so the #313 parity rationale is retired rather than // contradicted. + // + // No `jsx:` argument: the `.ts` grammar's enum carries `JsxText` in + // its shared externals, but no production emits it (#1483). + // + // `Interface` / `Enum` are the TS-only type-declaration keyword + // leaves, billed as the shared `class` is (#1552) and as C# and + // Java bill theirs; their `*_declaration` wrappers stay unlisted. + // + // #1557 adds the rest: `type`, `namespace`, `module`, `declare` and + // `global`. `Module2` is the `module` keyword leaf; the unsuffixed + // `Module` is the `module` declaration node that wraps it, so it + // stays unlisted. Each word is contextual, but used as a name + // (`let type = 1`, `module.exports`) the grammar emits an + // `identifier`, never the keyword leaf. The one exception is a + // binding named `type` in an export clause (`export { type }`), + // which the grammar recovers as the keyword leaf; `export_type_name` + // leaves that leaf unbilled. `type` also bills in `import type` / + // `export type`, where it is the same leaf. + // + // `TypeIdentifier` is the operand extra: every type *name* — a + // class, interface or alias name, a type parameter, an annotation + // (`x: Foo`), a generic (`Map`), the tail of `ns.T` — is this + // one leaf kind, and it was in neither arm before #1557, so TS lost + // every one while C#, Rust and the C family bill theirs. No + // classified node wraps it (`generic_type` and + // `nested_type_identifier` are unlisted), so it bills once. + // + // #1561 adds the type-level operator keywords, the siblings of the + // `as` / `typeof` / `extends` the macro already bills: `satisfies`, + // `keyof`, `infer`, the type-predicate `is` and `asserts`. Each is + // its keyword leaf; the `satisfies_expression`, `index_type_query`, + // `infer_type`, `type_predicate` and `asserts` wrappers stay + // unlisted, so one keyword bills once. `Asserts2` is the keyword + // token and the unsuffixed `Asserts` the node wrapping it, so + // `asserts a is T` bills `asserts` and `is` once each. Which of the + // two bills is unobservable — every node holds exactly one token and + // both render `asserts` — so the leaf is kept, as everywhere else + // here; listing both is what the tests catch. Used as names + // (`const keyof = 1`, `o.is`, a `satisfies` parameter) the words + // parse as identifiers. One valid spelling the pinned grammar gets + // wrong is `let satisfies = 1`: it reads `let` as an identifier and + // recovers a `satisfies_expression` around an ERROR, so that + // statement is mis-scored whatever this list holds. + // + // `generic_angles`: a type-argument or type-parameter list's `<` + // bills as the `<>` pair and its `>` as nothing (#1559). A `x` + // type assertion wraps a `type_arguments` node, so it is a pair too; + // a comparison's `<` / `>` sit under `binary_expression` and stay + // their own operators. impl_js_family_get_op_type!( Typescript, - op_extras: [QMARKDOT, PredefinedType], - operand_extras: [], + op_extras: [ + QMARKDOT, PredefinedType, Interface, Enum, Type, Namespace, Module2, Declare, Global, + Satisfies, Keyof, Infer, Is, Asserts2, + ], + operand_extras: [TypeIdentifier], predefined_void: PredefinedType, + generic_angles: GENERIC_ANGLES, + export_type_name: [Type, ExportClause, ExportSpecifier], ); get_operator!(Typescript); diff --git a/big-code-analysis-ast/src/lang_helpers.rs b/big-code-analysis-ast/src/lang_helpers.rs index f0b26341..9d3c6444 100644 --- a/big-code-analysis-ast/src/lang_helpers.rs +++ b/big-code-analysis-ast/src/lang_helpers.rs @@ -13,10 +13,12 @@ //! a helper *from* `metrics::cognitive` would make the parse layer depend //! on the metric layer, the inversion #1376 exists to remove. +pub mod bash; pub mod elixir; // Crate-private: this dialect's only helper is a kind table the three // classifiers in this crate share, and nothing outside names it. pub(crate) mod irules; +pub mod perl; pub mod python; pub mod tcl; // Crate-private for the same reason as `irules` above: the braced-word diff --git a/big-code-analysis-ast/src/lang_helpers/bash.rs b/big-code-analysis-ast/src/lang_helpers/bash.rs new file mode 100644 index 00000000..c3813554 --- /dev/null +++ b/big-code-analysis-ast/src/lang_helpers/bash.rs @@ -0,0 +1,67 @@ +//! Bash: the logical connectives and the string comparison of a +//! `[ … ]` / `[[ … ]]` test. + +use crate::Bash; +use crate::node::Node; + +/// The `&&` / `||` a `[ … ]` connective stands for: `Some(AMPAMP)` for +/// `-a` and `Some(PIPEPIPE)` for `-o`, when `op`, a child of `parent`, +/// is one. +/// +/// tree-sitter-bash gives every `-x` operator the one kind +/// `test_operator`, so the text decides. Only a binary position joins +/// two tests: the unary `-a FILE` (a file-exists test) has a +/// `unary_expression` parent and is a comparison like `-f FILE`. Every +/// expression kind is matched by name, not by `kind_id`, because the +/// grammar aliases `binary_expression` five times (grammar-dispatch §1). +/// +/// The `test` builtin spelling (`test "$a" = 1 -a "$b" = 2`) is out of +/// reach: the grammar parses its arguments as plain `word`s, so neither +/// its comparisons nor its connectives exist as nodes, and it scores as +/// one command. +#[must_use] +pub fn bash_test_connective(op: &Node, parent: &Node, code: &[u8]) -> Option { + if op.kind_id() != Bash::TestOperator as u16 || parent.kind() != "binary_expression" { + return None; + } + match code.get(op.start_byte()..op.end_byte())? { + b"-a" => Some(Bash::AMPAMP), + b"-o" => Some(Bash::PIPEPIPE), + _ => None, + } +} + +/// How many `=` string comparisons `test`, a `[ … ]` / `[[ … ]]` +/// `test_command`, holds. +/// +/// Both spellings of `=` are a `binary_expression`: `[ "$a" = 1 ]` +/// compares and `(( x = 1 ))` assigns. So an `=` compares exactly when +/// the test reaches it through expression nodes alone. The descent +/// passes through `binary_expression` because tree-sitter-bash nests `=` +/// wrongly — `[ "$a" = 1 -a "$b" = 2 ]` parses as +/// `"$a" = (1 -a "$b") = 2` — and through the unary and parenthesized +/// forms (`[[ ! ( "$a" = 1 ) ]]`). It stops at anything else, so the +/// assignment in `[[ $(( x = 1 )) -eq 1 ]]` is not counted. +/// +/// Counted from the test down rather than by climbing from each `=` to +/// its test: on the mis-nested spine a climb per `=` is `O(depth)`, and a +/// long `[ … = … -o … ]` made the walk quadratic. The descent is an +/// explicit stack, since that spine is as deep as the test is long. +#[must_use] +pub fn bash_test_eq_count(test: &Node) -> usize { + let mut count = 0; + let mut pending = vec![*test]; + while let Some(expression) = pending.pop() { + for child in expression.children() { + if child.kind_id() == Bash::EQ as u16 { + count += 1; + } else if matches!( + child.kind(), + "binary_expression" | "unary_expression" | "parenthesized_expression" + ) { + pending.push(child); + } + } + } + count +} diff --git a/big-code-analysis-ast/src/lang_helpers/elixir.rs b/big-code-analysis-ast/src/lang_helpers/elixir.rs index 2c040e3d..5fff724a 100644 --- a/big-code-analysis-ast/src/lang_helpers/elixir.rs +++ b/big-code-analysis-ast/src/lang_helpers/elixir.rs @@ -1,4 +1,5 @@ -//! Elixir: keyword `Call` identity (`def`, `defmodule`, `quote`, …). +//! Elixir: keyword `Call` identity (`def`, `defmodule`, `quote`, …), +//! and applied versus named operators. use crate::Elixir; use crate::node::{Ancestors, Node}; @@ -86,3 +87,66 @@ pub fn elixir_do_block_call_children<'a>( .flat_map(|do_block| do_block.children()) .filter(|stmt| stmt.kind_id() == Elixir::Call as u16) } + +/// The `binary_operator` that applies `op`, an operator token, or `None` +/// when the token is not applied there. +/// +/// An operator token also *names* the operator: `&==/2`, `&and/2` and +/// `Kernel.||(a, b)` hold it inside an `operator_identifier`, and the +/// single-token `atom` / `keyword` spell it too. None of those is a +/// `binary_operator` with operands to compare or chain, and under error +/// recovery the parent can be an `ERROR` with no operands, so every +/// decision metric keys on this parent rather than on the token (#1531, +/// #1534). A remote call that names the operator still *applies* it, +/// though; see [`elixir_remote_call_applies`]. +/// +/// Matched by rule name rather than by `kind_id`: the grammar aliases +/// `binary_operator` to three ids (`Elixir::BinaryOperator` through +/// `BinaryOperator3`), and a name comparison stays correct when a bump +/// adds a fourth (grammar-dispatch §1). +#[must_use] +pub fn elixir_applying_operator<'a>( + op: &Node<'a>, + ancestors: Ancestors<'a, '_>, +) -> Option> { + ancestors + .parent(op) + .filter(|parent| parent.kind() == "binary_operator") +} + +/// Whether `op`, an operator token, is the name of a remote call on +/// `Kernel` with arguments: `Kernel.||(a, b)`, or `x |> Kernel.||(y)`, +/// which the pipe rewrites to it. Such a call expands the `Kernel` macro +/// exactly as `a || b` does, so it short-circuits and decides as the +/// operator does, where a capture (`&Kernel.||/2`, a call with no +/// arguments) only names it. It has no `binary_operator`, so it carries no +/// `left` / `right` operands either. +/// +/// The receiver must spell `Kernel` (or `Elixir.Kernel`): a user module +/// may define `||/2` as an ordinary function, and `Foo.||(a, b)` neither +/// short-circuits nor decides. +/// +/// Matched by rule name: `dot` and `arguments` are aliased to several +/// ids each (grammar-dispatch §1). +#[must_use] +pub fn elixir_remote_call_applies<'a>( + op: &Node<'a>, + code: &[u8], + ancestors: Ancestors<'a, '_>, +) -> bool { + let mut up = ancestors.iter(op).map(|(node, _)| node); + up.next() + .is_some_and(|name| name.kind() == "operator_identifier") + && up.next().is_some_and(|dot| { + dot.kind() == "dot" + && dot.child_by_field_name("left").is_some_and(|receiver| { + matches!( + code.get(receiver.start_byte()..receiver.end_byte()), + Some(b"Kernel" | b"Elixir.Kernel") + ) + }) + }) + && up.next().is_some_and(|call| { + call.kind() == "call" && call.children().any(|child| child.kind() == "arguments") + }) +} diff --git a/big-code-analysis-ast/src/lang_helpers/perl.rs b/big-code-analysis-ast/src/lang_helpers/perl.rs new file mode 100644 index 00000000..de72ed6c --- /dev/null +++ b/big-code-analysis-ast/src/lang_helpers/perl.rs @@ -0,0 +1,258 @@ +//! Perl: telling the `and` and `not` operators, and the file tests, +//! from an auto-quoted key that the grammar lexes as one of them. + +use crate::Perl; +use crate::node::{Ancestors, Node}; + +/// Whether `and`, a Perl `and` token whose parent is `parent`, is the +/// low-precedence logical operator. +/// +/// FIXME(#1539 upstream): tree-sitter-perl 1.1.2 lexes an auto-quoted +/// bareword `and` as the keyword. Perl quotes a bareword before `=>` +/// and alone inside a hash subscript, so `(and => 1)` and `$h{and}` +/// hold a string key and no operator, yet the grammar emits two +/// shapes that read as one: +/// +/// - a subscript (`$h{and}`, `$r->{and}`, `$h{-and}`) recovers into an +/// `ERROR` node holding the token; +/// - a fat-comma key (`(and => 1)`, `{ and => 1 }`, `f(x => 1, and => +/// 2)`) becomes a `unary_expression` whose *first* child is the +/// token — the kind a real `$a and $b` also gets, with `$a` first. +/// +/// `and` is an infix operator, so in valid Perl the operator always has +/// a left operand. That is the whole test: the parent is one of the two +/// expression kinds and the token is not its first child, an `extra` +/// aside. +/// `or` and `xor` are unaffected — the grammar reads either as an +/// identifier in both positions. Remove this, and every call site, +/// once the pin carries the upstream fix. +#[must_use] +pub fn perl_and_is_operator(and: &Node, parent: &Node) -> bool { + matches!( + parent.kind_id().into(), + Perl::UnaryExpression | Perl::BinaryExpression + ) && first_non_extra(parent).is_some_and(|first| first.id() != and.id()) +} + +/// Whether `expr` is the `unary_expression` tree-sitter-perl builds +/// around an auto-quoted `and` key (`(and => 1)`): its first child, +/// comments aside, is the `and` token itself. See +/// [`perl_and_is_operator`]. +#[must_use] +pub fn perl_is_and_key_misparse(expr: &Node) -> bool { + first_non_extra(expr).is_some_and(|first| first.kind_id() == Perl::And as u16) +} + +/// Whether `not`, a Perl `not` token whose parent is `parent`, is an +/// auto-quoted hash key rather than the prefix operator. +/// +/// FIXME(#1541 upstream): the `not` sibling of #1539 — see +/// [`perl_and_is_operator`]. tree-sitter-perl 1.1.2 lexes a bareword +/// `not` before `=>` or alone in a hash subscript as the keyword, and +/// recovers into one of two shapes: +/// +/// - a subscript (`$h{not}`, `$r->{not}`) gives an `ERROR` node holding +/// the token; +/// - a fat-comma key (`(not => 1)`, `{ not => 1 }`) gives the +/// `unary_expression` a real `not 1` gets, with an `ERROR` wrapping +/// the `=>` as the token's next sibling; +/// - a fat-comma key after an earlier keyword key in the same list +/// (`f(and => 1, not => 2)`) gives a `not` applied to a zero-width +/// bareword the recovery invents, with the `=>` beside the whole +/// `unary_expression`. A real `not` always has a written operand. +/// +/// `and`'s test does not carry over: `not` is a prefix operator, so it +/// is the first child of its expression whether it is a key or not. The +/// recovery node is the only signal, so this reads it directly. A real +/// `not` that lands inside an `ERROR` for an unrelated reason is billed +/// as a key too; that takes input the grammar already failed to parse. +/// `or` is unaffected — the grammar reads it as an identifier in both +/// positions. Remove this, and every call site, once the pin carries the +/// upstream fix. +#[must_use] +pub fn perl_not_is_key<'a>(not: &Node<'a>, parent: &Node<'a>) -> bool { + parent.is_error() + || parent + .children_after(not) + // The parser marks the `ERROR` it wraps a skipped `=>` in + // as an extra, so only comments are passed over here. + .find(|child| child.is_error() || !child.as_tree_sitter().is_extra()) + .is_some_and(|next| { + (next.is_error() && next.is_child(Perl::FatComma as u16)) + || next.start_byte() == next.end_byte() + }) +} + +/// Whether `test`, a Perl `file_handle_operator` whose ancestor chain is +/// `ancestors`, is an auto-quoted `-bareword` key rather than a file +/// test. +/// +/// FIXME(#1545 upstream): tree-sitter-perl 1.1.2 lexes the leading `-X` +/// of a dash-prefixed bareword key as a file test. Perl auto-quotes +/// `-foo` before `=>` and alone in a hash subscript — the Tk, CGI and +/// Getopt option-hash idiom — so `$h{-foo}` and `(-text => 1)` hold the +/// strings `"-foo"` and `"-text"`, and no operator. The grammar emits +/// two file-test shapes, which this recognises: +/// +/// - `-` and a letter glued to more word characters (`-foo`) becomes a +/// file test on the bareword `oo`. Perl never reads that as a file +/// test, since one needs a non-word character after its letter, so +/// only the position decides: the test is the whole `key` of a hash +/// subscript, or the left of a `=>`. Elsewhere `-foo` is a negated +/// call or a string, and stays as the grammar reads it. +/// - a key that *is* a file test (`(-x => 1)`) swallows the `=>` and +/// the value, wrapping the `=>` in an `ERROR`. A real file test +/// cannot take a `=>` as its operand, so the `ERROR` decides — when +/// the `=>` is on the test's own line. Perl quotes `-x` only when +/// nothing but spaces and tabs separate it from the `=>`: across a +/// newline or a comment (`(-f # c⏎ => 7)`) it stays the file test on +/// `$_`, the list `(-f, 7)`, and the grammar recovers the same +/// `ERROR` regardless. A comment runs to the end of its line, so the +/// row alone tells the two apart. The `-and` / `-not` and `-foo` +/// keys are quoted across either, as Perl does. +/// +/// The subscript form of the second shape (`$h{-x}`) recovers into an +/// `ERROR` that drops the `-x` token altogether, so nothing is left to +/// bill. A key whose letter is no file-test letter (`-name`, `-height`) +/// is a third shape, a `-` applied to the bareword, which this does not +/// cover: it still bills the `-` and the word, and ABC counts the word +/// as a call, as every bareword key does. Remove this, and every call +/// site, once the pin carries the upstream fix. +#[must_use] +pub(crate) fn perl_file_test_is_dash_key<'a>( + test: &Node<'a>, + ancestors: Ancestors<'a, '_>, +) -> bool { + test.children() + // As in `perl_not_is_key`: the `ERROR` is an extra. + .find(|child| child.is_error() || !child.as_tree_sitter().is_extra()) + .is_some_and(|operand| { + if operand.is_error() { + operand.is_child(Perl::FatComma as u16) && operand.start_row() == test.start_row() + } else { + is_glued_word(&operand, test) && is_key_position(test, ancestors) + } + }) +} + +/// Whether `node` is the word of a `-bareword` key that the enclosing +/// file test bills whole, so it bills nothing of its own. `ancestors` +/// is `node`'s chain. +#[must_use] +pub fn perl_is_dash_key_word<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + ancestors.iter(node).next().is_some_and(|(test, above)| { + test.kind_id() == Perl::FileHandleOperator as u16 + && is_glued_word(node, &test) + && perl_file_test_is_dash_key(&test, above) + }) +} + +/// Whether `dash`, a Perl `-` token whose parent is `parent`, is the +/// sign of an auto-quoted `-and` or `-not` key (`$h{-not}`, +/// `(-and => 1)`), which the grammar splits into the `-` and the keyword +/// token. [`perl_and_is_operator`] and [`perl_not_is_key`] decide +/// whether the keyword is a key, and a key's sign is part of the string +/// whether or not a space separates the two. +#[must_use] +pub fn perl_dash_signs_keyword_key<'a>(dash: &Node<'a>, parent: &Node<'a>) -> bool { + // A sign is a prefix `-`, whose parent is the subscript's `ERROR` or + // a `unary_expression`. An infix `-` is a real subtraction even + // before an auto-quoted key (`(1 - and => 2)` subtracts the string), + // and is by far the commonest `-`, so it is answered first. + if parent.kind_id() == Perl::BinaryExpression as u16 { + return false; + } + // `$h{-not}` keeps the keyword beside the `-`; `(-not => 1)` nests + // it as the first child of the `-`'s operand. + next_non_extra(parent, dash).is_some_and(|next| { + keyword_is_key(&next, parent) + || first_non_extra(&next).is_some_and(|keyword| keyword_is_key(&keyword, &next)) + }) +} + +/// Whether `key`, an `and` / `not` key token whose ancestor chain is +/// `ancestors`, is signed by a `-` (`$h{-not}`, `(- and => 1)`), read off +/// the tree as [`perl_dash_signs_keyword_key`] reads it: the `-` sits +/// beside the key, or is the operator of the expression the key leads. +/// The bytes cannot say it: an infix `-` (`$x - and => 2`) or a comment +/// ending in one also precedes a key, and a comment can sit between a +/// sign and its key. +#[must_use] +pub fn perl_key_is_signed<'a>(key: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let mut up = ancestors.iter(key).map(|(ancestor, _)| ancestor); + up.next().is_some_and(|parent| { + let beside = parent + .children_before(key) + .find(|sibling| !sibling.as_tree_sitter().is_extra()); + if let Some(dash) = beside.filter(|node| node.kind_id() == Perl::DASH as u16) { + return perl_dash_signs_keyword_key(&dash, &parent); + } + up.next().is_some_and(|outer| { + outer.child_by_field_name("operator").is_some_and(|dash| { + dash.kind_id() == Perl::DASH as u16 && perl_dash_signs_keyword_key(&dash, &outer) + }) + }) + }) +} + +fn keyword_is_key(token: &Node, parent: &Node) -> bool { + match token.kind_id().into() { + Perl::And => !perl_and_is_operator(token, parent), + Perl::Not => perl_not_is_key(token, parent), + _ => false, + } +} + +// The width of a file-test operator token (`-e`, `-x`, …): the grammar +// lexes it as one hidden anonymous `/-[rwxo…]/` token, a `-` and one +// letter, so no node carries its end byte. +const FILE_TEST_TOKEN_LEN: usize = 2; + +// Whether `word` is the bareword the grammar split off a `-bareword` +// key: a word glued to the file-test token starts right after it. +fn is_glued_word(word: &Node, test: &Node) -> bool { + word.kind_id() == Perl::CallExpressionWithBareword as u16 + && word.start_byte() == test.start_byte() + FILE_TEST_TOKEN_LEN +} + +// Whether the `unary_expression` around `test` is the whole key of a +// hash subscript, which the grammar wraps in a `binary_expression` with +// no other child but comments, or is followed by a `=>`. +fn is_key_position<'a>(test: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let mut up = ancestors.iter(test).map(|(ancestor, _)| ancestor); + let mut key = up.next(); + let mut parent = up.next(); + if parent + .is_some_and(|p| p.kind_id() == Perl::BinaryExpression as u16 && non_extra_count(&p) == 1) + { + key = parent; + parent = up.next(); + } + key.zip(parent).is_some_and(|(key, parent)| { + matches!( + parent.kind_id().into(), + Perl::HashAccessVariableSimple | Perl::HashAccessVariable + ) || next_non_extra(&parent, &key) + .is_some_and(|next| next.kind_id() == Perl::FatComma as u16) + }) +} + +fn non_extra_count(node: &Node) -> usize { + node.children() + .filter(|child| !child.as_tree_sitter().is_extra()) + .count() +} + +// The sibling after `child` in `parent`, comments aside. Every key of +// a wide `(-a => 1, -b => 2, …)` list asks, so it is found without a +// scan from the list's start (#1545). +fn next_non_extra<'a>(parent: &Node<'a>, child: &Node<'a>) -> Option> { + parent + .children_after(child) + .find(|sibling| !sibling.as_tree_sitter().is_extra()) +} + +fn first_non_extra<'a>(node: &Node<'a>) -> Option> { + node.children() + .find(|child| !child.as_tree_sitter().is_extra()) +} diff --git a/big-code-analysis-ast/src/node.rs b/big-code-analysis-ast/src/node.rs index 23e37828..f7ce060b 100644 --- a/big-code-analysis-ast/src/node.rs +++ b/big-code-analysis-ast/src/node.rs @@ -506,6 +506,57 @@ impl<'a> Node<'a> { ChildrenWith { cursor, scan } } + /// The children after `child`, nearest first: what + /// `children().skip_while(|c| c.id() != child.id()).skip(1)` yields, + /// and nothing when `child` is not one of this node's children. + /// + /// That scan costs `child`'s index, so a walk that asks it of every + /// child of a wide list is quadratic in the list (#1545). This finds + /// `child` in `O(log n)` instead; see [`Node::children_before`]. + pub fn children_after(&self, child: &Node<'a>) -> impl Iterator> + use<'a> { + let mut cursor = self.cursor(); + let mut more = + self.seat_on_child(child, &mut cursor).is_some() && cursor.goto_next_sibling(); + std::iter::from_fn(move || { + let next = more.then(|| cursor.node())?; + more = cursor.goto_next_sibling(); + Some(next) + }) + } + + /// The children before `child`, nearest first, and nothing when + /// `child` is not one of this node's children. + /// + /// `child`'s index comes from [`Cursor::goto_first_child_for_byte`] + /// and each sibling from [`Node::child`], both `O(log n)` in the + /// child count, for the reason the first gives: a `repeat()` list is + /// stored as a balanced tree of hidden nodes. + pub fn children_before(&self, child: &Node<'a>) -> impl Iterator> + use<'a> { + let node = *self; + let index = self.seat_on_child(child, &mut self.cursor()).unwrap_or(0); + (0..index).rev().filter_map(move |i| node.child(i)) + } + + // Seats `cursor`, which sits on this node, on `child` and returns its + // index, or `None` when `child` is not one of this node's children. + fn seat_on_child(&self, child: &Node<'a>, cursor: &mut Cursor<'a>) -> Option { + if let Some(index) = cursor.goto_first_child_for_byte(child.start_byte()) + && cursor.node().id() == child.id() + { + return Some(index); + } + // The lookup passes over a zero-width child, which ends at the + // byte it starts at; only error recovery inserts one. + cursor.reset(self); + let mut index = 0; + let mut found = cursor.goto_first_child(); + while found && cursor.node().id() != child.id() { + found = cursor.goto_next_sibling(); + index += 1; + } + found.then_some(index) + } + /// A fresh cursor positioned on this node, for the traversals that /// reuse one cursor across many child scans /// ([`children_with`](Self::children_with)). @@ -1762,6 +1813,69 @@ mod tests { assert!(widest > 2, "fixture must contain a multi-child node"); } + /// `children_after` and `children_before` must yield, for every child + /// of every node, the raw `child(i)` walk on either side of it — and + /// nothing for a node that is not a child. + /// + /// The Kotlin fixture's recovery inserts a zero-width parameter name + /// (`vararg: Int`), which `goto_first_child_for_byte` passes over, so + /// it is the only input that reaches the scan fallback in + /// `seat_on_child`; the count of such children pins that it does. + #[cfg(feature = "kotlin")] + #[test] + fn children_after_and_before_match_the_child_walk() { + let code = b"fun f(vararg: Int, /* c */ b: Int) { g(1, 2, 3) }"; + let tree = Tree::new::(code); + let root = tree.get_root(); + + // Each row is (parent kind, child index, after ids, before ids), and + // a non-child row is (kind, after count, before count) against the + // root, so a failing diff names the node without a message argument. + let mut zero_width = 0; + let (mut observed, mut expected) = (Vec::new(), Vec::new()); + let (mut stray_observed, mut stray_expected) = (Vec::new(), Vec::new()); + for node in root.preorder() { + let raw = node.as_tree_sitter(); + let ids: Vec<_> = (0..raw.child_count() as u32) + .filter_map(|i| raw.child(i)) + .map(|c| c.id()) + .collect(); + for (index, child) in node.children().enumerate() { + zero_width += usize::from(child.start_byte() == child.end_byte()); + observed.push(( + node.kind(), + index, + node.children_after(&child) + .map(|c| c.id()) + .collect::>(), + node.children_before(&child) + .map(|c| c.id()) + .collect::>(), + )); + expected.push(( + node.kind(), + index, + ids[index + 1..].to_vec(), + ids[..index].iter().rev().copied().collect(), + )); + } + if node.id() != root.id() { + stray_observed.push(( + node.kind(), + root.children_after(&node).count(), + root.children_before(&node).count(), + )); + stray_expected.push((node.kind(), 0, 0)); + } + } + assert_eq!(observed, expected); + assert_eq!(stray_observed, stray_expected); + assert!( + zero_width > 0, + "fixture must hold an inserted zero-width child" + ); + } + /// The `O(1)` guard [`Ancestors::checked`] keeps on by default must /// accept every chain a walker really builds — over several grammar /// families, not just the one fixture a failure would surface in. diff --git a/big-code-analysis-bench/src/shapes.rs b/big-code-analysis-bench/src/shapes.rs index 5ea7bc92..4279b756 100644 --- a/big-code-analysis-bench/src/shapes.rs +++ b/big-code-analysis-bench/src/shapes.rs @@ -487,6 +487,53 @@ pub fn cpp_and_chain(depth: usize) -> String { format!("bool f(bool a){{ return {}a; }}\n", "a && ".repeat(depth)) } +/// Perl: `sub f { return g(-foo => 1, -foo => 1, …); }` — one call +/// whose argument list holds `width` dash-prefixed keys. +/// +/// The width shape for the #1545 dash-key workaround. tree-sitter-perl +/// reads each `-foo` as a file test, and the workaround asks whether a +/// `=>` follows it. Its first form found that `=>` by scanning the list +/// from its start, `O(width)` per key and so `O(width²)` per list: 7.5 s +/// for 4 000 keys in a release build. +#[must_use] +pub fn perl_dash_keys(width: usize) -> String { + format!( + "sub f {{ return g({}); }}\n", + vec!["-foo => 1"; width].join(", ") + ) +} + +/// Perl: `my @r = (/a/, /a/, …);` — one list of `width` bare matches. +/// +/// The width shape for the Perl ABC pattern-role check, which asks of +/// every pattern whether a `( … )` around it holds it as its value. Its +/// first form read the list's last operand to answer, scanning the whole +/// list per pattern: 7.7 s for 8 000 patterns in a release build. +#[must_use] +pub fn perl_pattern_list(width: usize) -> String { + format!("my @r = ({});\n", vec!["/a/"; width].join(", ")) +} + +/// Kotlin: `inline fun f(noinline p0: () -> Unit, …) {}` — one function +/// with `width` modified parameters. +/// +/// The width shape for #1558's recovery guard, which asks of every +/// parameter modifier whether the next token is an inserted name. Its +/// first form found that token by scanning the parameter list from its +/// start, quadratic in the parameter count. +#[must_use] +pub fn kotlin_noinline_params(width: usize) -> String { + let mut source = String::from("inline fun f("); + for i in 0..width { + if i > 0 { + source.push_str(", "); + } + let _ = write!(source, "noinline p{i:06}: () -> Unit"); + } + source.push_str(") {}\n"); + source +} + /// C++: `int f(int a){ return a + a + … + a; }`. /// /// The shape control for [`cpp_and_chain`]: the same left-nested @@ -1287,6 +1334,70 @@ pub const PROBES: &[Probe] = &[ grows with the width, so a walk that stopped \ counting them is visible in the value column too.", }, + Probe { + name: "halstead/perl-dash-keys", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Halstead], + reading: |m| m.halstead.total_operands(), + }, + render: perl_dash_keys, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1545's dash-key workaround asks of every `-foo` \ + whether a `=>` follows it. Finding that `=>` by \ + scanning the argument list from its start made the \ + walk quadratic in the key count: 1.85 s at 2 000 \ + keys and 7.45 s at 4 000 in a release build, against \ + 0.02 s and 0.03 s once the sibling is found by index. \ + ABC asks through the same helper. The reading is the \ + operand count, which grows with the width.", + }, + Probe { + name: "abc/perl-pattern-list", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc], + reading: |m| m.abc.conditions_sum(), + }, + render: perl_pattern_list, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "Perl ABC reads a pattern's bound or delimiter role on \ + the outermost `( … )` holding it as its value. \ + Deciding that from the list's last operand scanned \ + the list once per pattern: 1.9 s at 4 000 patterns \ + and 7.7 s at 8 000 in a release build, against \ + 0.015 s and 0.032 s once it looks forward from the \ + pattern. The reading is one condition per bare \ + match, so it grows with the width.", + }, + Probe { + name: "halstead/kotlin-noinline-params", + lang: LANG::Kotlin, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Halstead], + reading: |m| m.halstead.total_operators(), + }, + render: kotlin_noinline_params, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1558's recovery guard asks of every parameter \ + modifier whether the next token is an inserted name, \ + climbing to the parameter list. Scanning that list \ + from its start made the walk quadratic in the \ + parameter count: 0.73 s at 4 000 parameters and \ + 2.81 s at 8 000 in a release build, against 0.03 s \ + and 0.05 s once the sibling is found by index. The \ + reading is the operator count, which grows with the \ + width.", + }, Probe { name: "halstead/wide-distinct-fn", lang: LANG::Rust, diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 06e7abe5..19d7c6c5 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -116,10 +116,12 @@ each row attributed to the figure that introduces it. The short-circuit logical operators (`&&`, `||`, and per-language equivalents — Ruby `and` / `or`, Python `and` / `or`, Perl `and` / -`or` / `xor`, Lua `and` / `or`, Tcl `&&` / `||`, iRules `&&` / `||` / +`or`, Lua `and` / `or`, Tcl `&&` / `||`, iRules `&&` / `||` / `and` / `or`) do **not** contribute a condition on their own. Each non-comparison operand -contributes one instead, via the unary-conditional rule. The +contributes one instead, via the unary-conditional rule. Perl and +PHP `xor` evaluate both operands, so like `^` they are a value: the +slot holding one pays a single condition. The paper makes this explicit twice: 1. **Listing 2** annotates `(am >= 0 && am <= 0xF) ? '/' : 'C'` as @@ -145,21 +147,22 @@ application would over-count. | All languages | `default` / `_` wildcard arm excluded from the condition set | Fitzpatrick's Figure 2 lists `default`, but it falls through unconditionally — counting it would inflate `C` on every `switch` / `match` regardless of body. big-code-analysis omits it for every language (the Rust `_ =>` and Java `default:` arms included). | | Tcl | Chain-operand, bare-truthy and ternary slots wired; argument and `return` slots are not | Each operand of a `&&` / `\|\|` chain inside `expr {…}` counts as one condition, so `if {$a && $b}` reports two. `if` / `elseif` / `while` route their `expr {…}` predicate, so a bare-truthy `if {$a}` reports one and `if {!$a}` likewise — matching C's `if (a)` (#1180). A predicate written as a command substitution is a truthy test of that command's result, so `if {[somecmd]}` also reports one, and the redundant `if {[expr {$a \|\| $b}]}` idiom reports three (the chain's two operands plus the substitution). The argument and `return` slots remain unrouted: a negation reached only through a standalone `expr {…}` command still reports zero. | | iRules | Chain-operand, bare-truthy and ternary slots wired; argument and `return` slots are not | Each operand of a `&&` / `\|\|` / `and` / `or` chain counts as one condition (Rule 9), so `if {!$a && !$b}` reports two. iRules also recognises the word-form string-match comparators (`contains`, `starts_with`, `ends_with`, `equals`, `matches`, …) that Tcl lacks (Tcl's `eq` / `ne` / `in` / `ni` are shared). Bare-truthy and ternary slot routing matches its Tcl sibling exactly (#1180) — see that row for the shared detail, including the argument and `return` slots that remain unrouted. | -| All Phase 2 languages (Java, Groovy, C#, Rust, Go, JavaScript, TypeScript, TSX, Mozjs, PHP, C, C++, Objective-C, Mozcpp, Python, Perl, Lua) | `if (true) {}`, `m(!a, !b)`, `return !x` count their operand(s) | Phase 2B routes `if` / `while` / `do-while` / argument-list / `return` slots through the same walker, so the rule applies uniformly across decision-bearing positions. A bare `return x` continues to report zero — Fitzpatrick treats an identifier in a return slot as a value, not a unary conditional. C# is the one exception to the argument-list half of the rule: its grammar wraps each argument in an `argument` node that the walker does not descend, so `M(!a, !b)` reports zero where every other language in this row reports two. `if (true)` and `return !x` behave as stated. | +| All Phase 2 languages (Java, Groovy, C#, Rust, Go, JavaScript, TypeScript, TSX, Mozjs, PHP, C, C++, Objective-C, Mozcpp, Python, Perl, Lua) | `if (true) {}`, `m(!a, !b)`, `return !x` count their operand(s) | Phase 2B routes `if` / `while` / `do-while` / argument-list / `return` slots through the same walker, so the rule applies uniformly across decision-bearing positions. A bare `return x` continues to report zero — Fitzpatrick treats an identifier in a return slot as a value, not a unary conditional. C# wraps each argument in an `argument` node, so the walker reads the value after any named-argument label or `ref` / `out` / `in` keyword: `M(!a, !b)` and `M(name: !a, !b)` report two, as do the base calls `: base(!a, !b)` and `class S(bool a, bool b) : Base(!a, !b)`. Before #1537 it read the wrapper instead, and every C# argument reported zero. An indexer argument (`o[!a]`) is not a call argument and reports zero in C#, as Java's array index does. | | Ternary slots: Java, Groovy, C#, C, C++, Objective-C, Mozcpp, JavaScript, TypeScript, TSX, Mozjs, PHP, Perl, Ruby, Python, Tcl, iRules | `a ? !b : !c` counts its condition and both branch operands | The same walker also runs over a ternary's three operand slots, so `a ? !b : !c` reports 4 (the `?` plus three unary conditions) rather than 1. Tcl and iRules reach the same 4 without grammar fields: their `ternary_expr` exposes none, and `_expr` inlines `( … )`, so the slots are located relative to the `?` and `:` tokens instead of by index (#1180). Python arrives at the same total by a different route: `not` operands are counted by the `NotOperator` rule wherever they appear, so only the condition slot is routed through the walker and `(not b) if a else (not c)` likewise reports 4. Languages with no ternary (Rust, Go, Kotlin, Lua, Elixir) are unaffected. | | `for` header condition slot: Java, Groovy, C#, C, C++, Objective-C, Mozcpp, JavaScript, TypeScript, TSX, Mozjs, PHP, Go, Perl | `for (; a; )` counts one condition; `for (;;)` counts none | The loop header's condition is a decision-bearing slot like an `if` predicate, so `for (; a; )`, `for (; !a; )` and `for (; (a); )` each report one. A comparison-shaped condition (`i < n`) is already counted by its own operator arm and is not double-counted. Every one of these grammars except Go exposes a `condition` field on `for_statement` (Perl's sits on its C-style `for_statement_1`), so the slot is read by field name rather than by child index; Go's field sits on the nested `for_clause`, and its outer header slot is located structurally (first named child that is neither the body nor a comment). An **empty** condition counts zero in every language: Fitzpatrick's rules count conditional operators and unary conditions that are present, and an omitted test is not a decision. Java and Groovy previously scored `for (;;)` as one vacuously-true condition and no longer do. Tcl, iRules and Bash are command-dispatched grammars whose loop headers carry no boolean-expression slot to route (grammar-dispatch §9) (#1276). | | Ruby | Bare-predicate `if` / `unless` / `while` / `until` (block and modifier forms) count one condition | Idiomatic Ruby favours bare predicates (`if flag`, `x if flag`); counting the condition slot keeps ABC conditions at or above Ruby's cyclomatic decision count (the alignment enforced across the other languages). A comparison (`if a == b`) or `&&` / `\|\|` chain in the predicate is counted by its own operator / walker arm and is not double-counted. Each `&&` / `\|\|` / `and` / `or` operand is scored by the same rule — once, unless its own arm already counts it — so `a && self`, `a && Foo::Bar` and `a && -b` score 2 like `a && b`, wherever the chain is written (#1529). | | Python, JavaScript, TypeScript, TSX, Mozjs, C, C++, Objective-C, Mozcpp, Java, C#, Rust, PHP, Perl, Lua, Groovy, Kotlin, Tcl, iRules | A boolean slot pays one condition unless something inside its predicate already counts it | Every `if` / `while` / `do` / `for` condition, ternary condition, guard and `&&` / `\|\|` operand scores the way Ruby's has since #1520: one condition, unless the predicate — seen through parentheses, a negation, a cast or a plain assignment — is a comparison, chain, ternary or type test that its own rule counts. Before #1526 these slots paid only for a predicate from a fixed list of operand kinds, so `if (-x)`, `if (x + 1)`, `if (this)`, `if (*p)`, `if (flags & k)`, `if (y = x)` and `if (sizeof x)` each scored 0 against a cyclomatic decision of 1, as did Java's, C#'s and Rust's `if (a & b)` and a boolean `switch` / `match` expression. The same rule stops a wrapped comparison paying twice: `if ((int)(x > 1))`, Python's `if (y := x > 1):` and Python's `if not (x > 0):` score 1, not 2, as Java's `if (!(x > 0))` always has. Python's `elif`, PHP's `elseif`, Lua's `elseif` and Perl's `elsif` now score like Ruby's `elsif` and Java's `else if`: one for the `else` plus the predicate slot, so `elif b` no longer scores one below `elif x > 0`. Kotlin joined in #1533, so the slot pays for a `when` it holds, as it does for Java's `switch`, C#'s `switch` expression and Rust's `match`: `if (when (a) { else -> true })` scores 1 (it scored 0) and `if (when (x) { 1 -> true; else -> false })` 2 (it scored 1), its entry paying 1 and the slot 1. Go still pays a slot through an operand list (#1526). | -| Bash | `if` / `elif` / `while` and each non-wildcard `case` arm count one condition | A Bash predicate is a *command*, so the branch keyword itself — not an embedded boolean expression — is the condition signal. Each matches a Bash cyclomatic decision; the bare `*)` case arm (the analogue of `default:`) is excluded, mirroring the cyclomatic standard count. The arithmetic ternary `$(( a ? b : c ))` therefore contributes nothing: it carries no branch keyword, so it falls outside the rule set rather than through a gap in it. | +| Bash | `if` / `elif` / `while` and each non-wildcard `case` arm count one condition | A Bash predicate is a *command*, so the branch keyword itself — not an embedded boolean expression — is the condition signal. Each matches a Bash cyclomatic decision; the bare `*)` case arm (the analogue of `default:`) is excluded, mirroring the cyclomatic standard count. The arithmetic ternary `$(( a ? b : c ))` counts one, as the C-family ternary does (#1268). Each comparison inside a test pays one more: `==`, `!=`, the `-eq` / `-z` / `-f` family, and `=`, which compares in `[ … ]` / `[[ … ]]` and assigns in `(( … ))`. The `[ … ]` connectives `-a` / `-o` join two tests as `&&` / `\|\|` do, and pay nothing, so `[ X -a Y ]`, `[ X ] && [ Y ]` and `[[ X && Y ]]` score alike in every decision metric; a bare `test X -a Y` is all plain words to the grammar and scores none of them (#1536). | | Kotlin | `try` counts a condition alongside `catch` | Fitzpatrick counts both keywords, and Java / C# / C++ / Groovy already count both; Kotlin previously counted only the catch block. | | C#, Java, Rust, Python, Ruby, Elixir | A pattern-match guard is a condition slot, scoring one however it is spelled | A guard is a branch: the pattern can match while the guard fails. The guard's expression is scored exactly as an `if` condition is — one for a call, type test, attribute or bare identifier, one via the operator arm for a comparison — so every spelling agrees, and a compound guard (`when a > 1 && b < 2`) keeps its sub-structure rather than collapsing to one. Before this, ABC scored whatever operator happened to sit inside, so `when x > 2` counted one and the equivalent `when IsEven(x)` counted none. The spellings, per language: C# `when_clause` on a switch arm or `case` section and `catch_filter_clause` on a `catch` (#1422); Java 21's `guard` on a pattern-switch label, in both the arrow and colon forms; Rust's `match_pattern` guard; Python's `case … if g:`; Ruby's `if_guard` / `unless_guard` on a `case … in` arm; Elixir's `when` operator on a `stab_clause` head or a `def` / `defguard` head (#1454). The same change made the guard a **cyclomatic** decision wherever it was not already one — Java, Ruby and Elixir — and excluded Elixir's typespec `when` (`@spec f(a) :: a when a: integer`), which spells the same token as a guard but is type syntax. Groovy and Kotlin are absent because neither pinned grammar has a guard production at all: Kotlin 2.1 guard syntax does not parse, so there is nothing to classify (#1454). | | Kotlin | A subject-less `when` arm scores through the predicate slot; a subject-ful arm scores per entry | A subject-less arm's condition (`when { x > 5 -> … }`) is an ordinary boolean expression, compiled as an `if` predicate, so the comparison inside it is already counted by the operator arms and a blanket per-entry increment double-counted it. A subject-ful arm (`when (x) { in 1..2 -> … }`) lists a pattern rather than an independent boolean expression, so the implicit `subject == pattern` is a decision nothing in the source spells and the entry itself pays for it (#1421). | | Ruby | A subject-less `case` scores each `when` pattern as an `\|\|` operand; a subject-ful `case` scores per clause | A subject-less clause (`case; when x > 5`) lists ordinary boolean expressions, evaluated like an `if` predicate, and several patterns (`when a, b`) are an implicit `\|\|`. Each pattern therefore scores once unless its own operator already counted it, so `when x > 5` scores 1 like `if x > 5`, and `when a, b` scores 2 like `if a \|\| b`. Before this, a blanket per-clause increment counted a comparison twice. A subject-ful clause (`case x; when 1`) matches each pattern with an implicit `pattern === x` that nothing in the source spells, so the clause pays once (#1453). | | Go | A tagless `switch` scores each case expression as an `\|\|` operand; a tagged `switch` scores per case | A tagless switch (`switch { case x > 5: }`, with or without an init statement) is `switch true`: each case expression is an ordinary boolean scored by the same slot an `if` predicate uses, and a list (`case a, b:`) is an implicit `\|\|`. So `case x > 5:` scores 1 like `if x > 5`, and `case a, b:` scores 2 like `if a \|\| b`. Before this, a blanket per-case increment counted a comparison twice. A tagged case (`switch x { case 1: }`) compares with an implicit `x == 1` that nothing in the source spells, so the case pays once; type-switch and `select` cases are unchanged (#1523). | -| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive` (its `after` timeout included), of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct, and a `with` with no `else` now scores nothing, level with cyclomatic. The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). | -| C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these `<` / `>` as an operator, and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | +| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive` (its `after` timeout included), of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). | +| C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these brackets as an operator (a generic list's as one `<>` pair since #1559), and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | -| C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | +| C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | +| Perl | A comparison's right operand is not a condition, although the grammar parses it as one | tree-sitter-perl 1.1.2 lets a chaining comparison (`<`, `>`, `<=`, `>=`, `lt`, `gt`, `le`, `ge`, `==`, `!=`, `eq`, `ne`) take everything of lower precedence to its right as its operand, so `$a > $b ? 1 : 2` parses as `$a > ($b ? 1 : 2)` and `$a > $b && $c` as `$a > ($b && $c)`. The ternary's condition slot and the chain's first operand then held `$b`, which scored as a condition of its own: one above the JavaScript twin for each misparsed comparison. The walker recognises the shape and leaves that operand to the comparison, so both spellings score what the parenthesised `($a > $b) ? 1 : 2` and `($a > $b) && $c` do. An explicit `$a > ($b ? 1 : 2)` is unaffected. `<=>`, `cmp`, `=~` and `!~` do not chain and parse correctly. The workaround goes when the grammar is fixed upstream (#1273). | #### Where a constructor call lands {#abc-constructor-attribution} @@ -334,8 +337,8 @@ where the cognitive load jumps, so the second operator earns its own token: where a language spells conjunction and disjunction both ways (C++, Ruby, PHP, Perl, Elixir, iRules), `a && b and c` is one sequence. Precedence still decides what nests where, so Ruby's -`a || b and c` is `(a || b) and c` and scores two. `xor`, `??` and -Perl's `//` are operations of their own. +`a || b and c` is `(a || b) and c` and scores two. `??` and Perl's +`//` are operations of their own; `xor`, like `^`, is no sequence. big-code-analysis exports the per-function structural score along with the file-wide `sum`, `min`, `max`, and a per-function `average`. @@ -392,6 +395,14 @@ Sonar ecosystem. levels up happened to be a closure. Python cannot express the shape at all: a `def` is a statement and a lambda body is a single expression. +- **Rust**'s `let`-`else` (`let Some(a) = y else { continue };`) + postdates the specification. It is scored as the `if let` it + replaces: `+1` plus the current nesting, with everything under it, + the `else` block included, one level deeper. Its `else` token does + not pay again. A `for` holding it scores 3, the same as one holding + `if let Some(a) = y { … }`. Before #1548 the `let`-`else` paid a flat + `+1` through its `else` token whatever its depth, so that `for` + scored 2. - **Python** charges a boolean operator an extra `+1` for each enclosing `lambda`, on top of the `+1` the boolean sequence itself earns. No other language does this. Only the outermost operator @@ -495,6 +506,17 @@ echoes the long-standing `-m` mode from Terry Yin's many readers will first have seen it.) Both numbers are exported side by side; pick one and be consistent. +Only alternatives collapse. A sequence of tests is still one decision +per test in both variants. Elixir's `with` is the case where the two +meet: each `pattern <- expr` clause is a separate test, scored once in +both variants, as Rust's `if let` and `?` are. The clauses of its +`else` are alternatives, so modified scores them once, and only when +one of them is a decision: `else _ -> 0` selects nothing, like a plain +`else`, and is free in both variants (#1535). Rust's `let`-`else` +(`let Ok(a) = g(x) else { return 0 };`) is one such test too. It binds +or diverges, so it scores one in both variants, as its `if let` twin +does. Before #1542 it scored nothing. + ### Counting Rust's `?` operator By default Rust's `?` operator (the `try_expression` grammar node) @@ -587,11 +609,14 @@ notation for the same four counts. big-code-analysis records these four numbers in `src/metrics/halstead.rs` per function and per file. The per-language trait classifies tokens as operator vs. operand on a token-by-token -basis; the rules deliberately exclude pure layout punctuation like -parentheses and statement separators, which is why the Halstead -totals are *not* the same as the Tokens count. - -Six classification rules are worth knowing because they are choices +basis. Punctuation counts as an operator: a bracket pair is one +operator, billed once at its opener and listed by `bca ops` as `()`, +`[]` or `{}`, and `,`, `;`, `.` and `:` count once per occurrence. +Closing brackets, comments and a literal's own delimiters count as +nothing, which is why the Halstead totals are *not* the same as the +Tokens count. + +The classification rules below are worth knowing because they are choices rather than consequences, and because several grammars spell the tokens involved the same way they spell real operators: @@ -600,7 +625,10 @@ tokens involved the same way they spell real operators: and a Perl or Ruby pattern each contribute *one operand* — the literal — and no operator for the punctuation around it. Otherwise the score would move with the author's choice of delimiter, which - says nothing about the code. Tcl and iRules need one extra step to + says nothing about the code. The same holds for punctuation the + grammar spells *inside* a literal: Perl's bare `/[a]#/` bills no `[]` + or `#`, matching `m/[a]#/`, whose text the grammar does not split + into tokens (#1549). Tcl and iRules need one extra step to honour this, because their grammars spell a braced literal and a braced script body the same way: whether `{a b}` is a block or a quoted value depends on the command it is passed to, so the @@ -659,9 +687,13 @@ tokens involved the same way they spell real operators: belongs to it — so the list would be incomplete by construction and would make three behaviours where there are two. - **A string-interpolation opener is not an operator.** `"{$x}"` in - PHP, `"#{x}"` in Ruby and Elixir, `"${x}"` in Kotlin and Groovy and - `$"{x}"` in C# all count the interpolated expression's own operators - and nothing for the opener itself. + PHP, `"#{x}"` in Ruby and Elixir, `"${x}"` in Kotlin and Groovy, + `$"{x}"` in C# and `f"{x}"` in Python all count the interpolated + expression's own operators and nothing for the opener itself. The + `:` that opens a Python format spec (`f"{x:>10}"`) counts nothing + either, because the spec after it is string content (#1486). Nor + does the self-documenting `=` of `f"{x=}"`, which prints the + expression's text and assigns nothing (#1551). - **A delimiter around *syntax* is an operator, whatever it delimits.** The rule above is about a literal's own punctuation, and it stops there. Halstead counts the vocabulary a program is written in and has @@ -674,6 +706,76 @@ tokens involved the same way they spell real operators: tokens from `conditions`: ABC asks whether a branch is taken, and these are not branches; Halstead asks what alphabet the source uses, and these are part of it. Expect the two metrics to disagree here. + The same reading bills Python's `(`, `[`, `{`, `;` and block, + slice, dict and annotation `:`, and Rust's annotation, field, bound + and label `:`, which is a separate operator from `::`. Both languages + left these out until #1486, so their Halstead scores rose with it. + Python's `class` keyword followed in #1550: it is now billed like + `def`, as Ruby, Kotlin, C# and PHP bill their `class`. Java, Groovy, + the JavaScript family, C, C++ and Objective-C followed in #1552 with + their type-declaration keywords: `class`, `struct`, `union`, `enum`, + `interface`, `record`, `trait`, `@interface`, `@implementation` and + `@protocol`. Java's `C.class` is not billed: it stays one operand. + Kotlin's `interface` followed in #1556, and since #1558 Kotlin bills + every modifier, as C# does (Java and Groovy bill `abstract` and + `final` but no visibility keyword or `static`, #1565): the `enum` of + `enum class`, `data`, `sealed`, the visibility keywords, `open`, + `override`, `suspend`, `reified`, `out` and the rest each count once, + as the keyword and never as its `modifiers` wrapper. The same words + used as names bill no operator: where the grammar parses the name + (`val open = 1`) it is an operand, and where it cannot (`open = !open` + at the start of a statement, `val suspend = 1`, a parameter named + `vararg`) the word bills nothing. #1557 added C++'s `namespace`, `template`, `typename`, + `typedef` and `using` (C and Objective-C bill `typedef` too), so + `template ` and `template ` each bill their + keyword and score alike. It also added TypeScript's `type`, + `namespace`, `module`, `declare` and `global`, and made every + TypeScript type name an operand (`x: Foo`, `interface I`, + `Map`), as C#, Rust and the C family already did. Java followed + in #1560: `Foo f`, `implements I`, `List` and each segment of + `java.util.List` bill their names as operands. A type inside a class + literal (`Foo.class`) does not, since the literal is already one + operand, and neither does the inferred type `var`, which C# bills + nothing for either. #1561 made TypeScript's type-level operator + keywords operators, as their value-level siblings `as`, `typeof` + and `extends` already were: `satisfies`, `keyof`, `infer`, and the + `is` and `asserts` of a type predicate. +- **A generic list's angle brackets are one bracket pair.** The `<` + and `>` around type arguments and type parameters count as one `<>` + operator, billed at the `<` as `()` is at the `(`, and the `>` counts + nothing (#1559). That covers `Vec`, `fn f`, a turbofish + `h::()`, `for<'a>` and `use` in Rust; C++'s + `template ` and `std::vector`; and the type-argument + and type-parameter lists of Java, Groovy, C#, Kotlin, TypeScript and + TSX. Objective-C has four generic lists: `NSArray`, + `id`, the protocol list in + `@interface A : NSObject `, and a type passed as a call + argument, `f(NSArray)`. A comparison's `<` and `>` and a shift's `<<` and `>>` stay + separate operators, and a C++ `>>` that closes two template lists + parses as two `>`, so it bills nothing. The rule follows the parse: + where tree-sitter-cpp reads comparisons as a template argument list, + as in `x = a < b || c > (d)`, their brackets bill as `<>` too, since + only name lookup tells that from a template call spelled the same. + The comparisons Objective-C and C# misread as a call argument's + generic list, `g(ad)` and `F(a < b, c > d)`, keep `<` and `>`. + The exception is C#'s `>=` spelling, `F(a < b, c >= d)`, which the + grammar parses as an assignment and which still bills `<>` and `=`. Go's generics use square + brackets and already billed `[]`. Angle brackets around anything + other than a generic list still bill `<` and `>`: a Rust + `::` path, a C# `delegate*` pointer type and + a Kotlin `super`, as well as the JSX, Lua and Perl constructs + above. So does a Kotlin generic call with arguments, `id(a)`, + which tree-sitter-kotlin-ng parses as two comparisons (#1394), and a + generic inside a Rust macro call, `vec![Vec::::new()]`, whose + token tree tree-sitter-rust leaves unparsed; a `>>` closing two lists + there bills as the shift. + Before #1559 each bracket billed under the comparison's own entry, + so `Vec` and `a < b` shared one. Every generic now costs + one operator occurrence less, so `N1` falls in any code that uses + generics. `n1` rises by one where a comparison `<` or `>` remains + beside the new `<>`, so `halstead.effort` falls in most files and + rises in the rest; on the C++ and Rust corpora it fell in 87% of the + files that moved, and MI rose in most of them. - **An alternative spelling is an operator of its own.** C++ spells `&&` equally as `and`, `!=` as `not_eq` and so on through all eleven [alternative tokens](https://en.cppreference.com/w/cpp/language/operator_alternative). @@ -692,12 +794,50 @@ tokens involved the same way they spell real operators: `? super T` in Java and Groovy (#1380). - **A keyword's role follows its text, not its node kind.** C#'s grammar parses the bare `this`, `ref`, `out`, `in`, `scoped` and - `readonly` of a parameter as a `modifier` node, a different kind - from the one the same keyword has anywhere else. Each keeps the - role its text has elsewhere: an extension method's `this Foo f` is - an operand, and the `ref` in `M(ref int a)` is the same operator as - the `ref` in the call `M(ref x)`, adding to `total_operators` but - not to `unique_operators` (#1418). + `readonly` of a parameter, and the `static` / `async` of a lambda, + as a `modifier` node, a different kind from the one the same + keyword has anywhere else. Each keeps the role its text has + elsewhere: an extension method's `this Foo f` is an operand, the + `ref` in `M(ref int a)` is the same operator as the `ref` in the + call `M(ref x)`, and `static (int x) => x` bills the same `static` + as `static void M()`. A repeated operator keyword adds to + `total_operators` but not to `unique_operators` (#1418, #1482). +- **JSX element text is an operand.** In JavaScript, Mozjs and TSX, + the `hi` of `

hi

` counts the way a string's contents do. It is + keyed on its text with leading and trailing whitespace trimmed, so a + line of text at two indentation depths is one operand, and the + spaces between two tags on one line count as nothing. Whitespace + *inside* a multi-line text is kept. A character reference (`&`) splits the text + into separate operands (`a & b` is three). Inside an attribute + string, the reference is part of that string's one operand (#1483). +- **An overloaded operator's name is an operator.** The `+` of + `operator +` was always one, and the names a grammar spells as whole + tokens now agree: C#'s `operator true` / `operator false`, and C++'s + `operator[]`, `operator()`, `operator""_x` and `operator co_await`. + The `operator` keyword is an operator in both languages, as is C++'s + `co_await` in an expression. C++'s `[]` and `()` names are tokens of + their own, so `bca ops` lists `operator[]`'s `[]` beside a + subscript's `[]` as a second operator, as Ruby does for `def [](i)` + (#1296). C++'s coroutine keywords `co_return` and `co_yield` are + operators as `return` is (#1547). +- **Only written tokens count.** When tree-sitter cannot parse a + construct, its error recovery may insert zero-width tokens the source + does not contain: a `;`, a `)`, an identifier. None of them is billed + in either half. Before #1546 an inserted identifier became an operand + whose text is the empty string. One case is a C++ destructor called + through a template name (`p->~T()`), which tree-sitter-cpp cannot + parse. Another is a statement-position `typename` cast, which made + every inserted `;` a second `;` operator. +- **A named operator is one operand.** Elixir names an operator in a + capture (`&==/2`, `&and/2`) and as the function of a remote call + (`Kernel.||(a, b)`), and its grammar wraps the token in an + `operator_identifier`. That wrapper is the operand, as `foo` is in + `&foo/2`, so the token inside it adds no operator; `&==/2` scores + what `&foo/2` does. A captured `&&` / `||` / `and` / `or` is no + decision either (#1534), but the remote call applies the operator, + so `Kernel.||(a, b)` is a cyclomatic decision as `a || b` is. This + differs from a C# `operator <`, whose `<` has no wrapper and so stays + the operator the rule above makes it. ### Derived metrics @@ -1019,7 +1159,10 @@ ones that have accreted "just one more parameter" feature flags. **NExits** counts the number of distinct exit points from a function — every explicit `return`, every `throw` / `raise`, and -(in Rust) every `?` early-return. The implicit fall-through return at +(in Rust) every `?` early-return. A generator's `yield` counts in +Python, the JavaScript family, C# and PHP, because it hands control +back to the caller, and so does C++'s `co_yield`; C++'s `co_return` is +a `return` (#1547). The implicit fall-through return at the end of a function is **not** counted; only explicit exits are (see issue #243). diff --git a/big-code-analysis-cli/tests/check/check_explain_threshold.rs b/big-code-analysis-cli/tests/check/check_explain_threshold.rs index 0e32cb35..8fde35c6 100644 --- a/big-code-analysis-cli/tests/check/check_explain_threshold.rs +++ b/big-code-analysis-cli/tests/check/check_explain_threshold.rs @@ -523,16 +523,16 @@ fn soft_limit_is_derived_by_metric_direction() { /// expression, `breaches_limit(v.value, ceiling, v.lower_is_worse)`. /// /// The test above cannot reach it: `mi.original=20` is a floor that every -/// one-line fixture function (all measure ≈147-149) clears, so its +/// one-line fixture function (all measure ≈145-148) clears, so its /// population is empty at both tiers and the partition never runs. /// Hard-coding `false` there — the pre-#1166 "higher is worse" assumption /// — failed none of the suite's 5053 tests. /// -/// A floor of `147.5` splits the fixture's three measured values -/// (146.9456 ×3, 147.7445 ×6, 148.655 ×20): three sit below it and -/// breach, all 29 sit below the derived `155.264` soft floor. Read with +/// A floor of `146` splits the fixture's three measured values +/// (145.7045 ×3, 146.5454 ×6, 147.5134 ×20): three sit below it and +/// breach, all 29 sit below the derived `153.684` soft floor. Read with /// the direction inverted the hard tier would instead collect the 26 -/// *above* 147.5, so the two readings share no count. +/// *above* 146, so the two readings share no count. /// /// Both metrics are explained in one run, and both carry a *non-empty* /// population — which is the second thing this pins. Every other @@ -551,7 +551,7 @@ fn a_lower_is_worse_metric_partitions_offenders_by_direction() { "--paths", "lib.rs", "--explain-threshold", - "mi.original=147.5", + "mi.original=146", "--explain-threshold", &candidate, ])); diff --git a/big-code-analysis-web/src/web/server_tests.rs b/big-code-analysis-web/src/web/server_tests.rs index 33e868e7..1bb9c1d3 100644 --- a/big-code-analysis-web/src/web/server_tests.rs +++ b/big-code-analysis-web/src/web/server_tests.rs @@ -743,26 +743,26 @@ async fn test_web_metrics_json() { "cognitive": {"sum": 0, "value": 0, "average": 0.0, "min":0, "max":0}, "nargs": {"function_args": 0, "function_args_average": 0.0, "closure_args": 0, "closure_args_average": 0.0, "total": 0, "value": 0, "average": 0.0, "closure_args_max": 0, "closure_args_min": 0, "function_args_max": 0, "function_args_min": 0}, "nexits": {"sum": 0, "average": 0.0,"min":0,"max":0}, - "halstead": {"bugs": 0.000_942_552_557_372_941_4, - "difficulty": 1.0, - "effort": 4.754_887_502_163_468, - "length": 3, - "estimated_program_length": 2.0, - "purity_ratio": 0.666_666_666_666_666_6, - "level": 1.0, + "halstead": {"bugs": 0.002_712_967_490_108_627, + "difficulty": 2.0, + "effort": 23.219_280_948_873_62, + "length": 5, + "estimated_program_length": 8.0, + "purity_ratio": 1.6, + "level": 0.5, "total_operands": 1, - "total_operators": 2, - "vocabulary": 3, - "time": 0.264_160_416_786_859_36, + "total_operators": 4, + "vocabulary": 5, + "time": 1.289_960_052_715_201, "unique_operands": 1, - "unique_operators": 2, - "volume": 4.754_887_502_163_468}, + "unique_operators": 4, + "volume": 11.609_640_474_436_81}, "loc": {"cloc": 1, "ploc": 2, "lloc": 1, "sloc": 4, "blank": 1, "cloc_average": 0.5, "ploc_average": 1.0, "lloc_average": 0.5, "sloc_average": 2.0, "blank_average": 0.5, "cloc_min": 0, "ploc_min": 2, "lloc_min": 1, "sloc_min": 2, "blank_min": 0, "cloc_max": 1, "ploc_max": 2, "lloc_max": 1, "sloc_max": 4, "blank_max": 1}, "nom": {"functions": 1, "closures": 0, "functions_average": 0.5, "closures_average": 0.0, "total": 1, "average": 0.5, "closures_min": 0, "closures_max": 0, "functions_min": 0, "functions_max": 1}, "tokens": {"tokens": 6, "average": 3.0, "min": 0, "max": 6}, - "mi": {"original": 139.974_331_558_152_1, - "sei": 176.151_664_509_301_33, - "visual_studio": 81.856_334_244_533_39}, + "mi": {"original": 135.332_485_042_474_81, + "sei": 169.454_895_560_566, + "visual_studio": 79.141_804_118_406_33}, "abc": {"assignments": 0, "branches": 0, "conditions": 0, "magnitude": 0.0, "value": 0.0, "assignments_average": 0.0, "branches_average": 0.0, "conditions_average": 0.0, "assignments_min": 0, "assignments_max": 0, "branches_min": 0, "branches_max": 0, "conditions_min": 0, "conditions_max": 0}, "wmc": {"class_wmc_sum": 0, "interface_wmc_sum": 0, "total": 0}, "npm": {"class_npm_sum": 0, "interface_npm_sum": 0, "class_methods": 0, "interface_methods": 0, "class_coa": 0.0, "interface_coa": 0.0, "total": 0, "total_methods": 0, "coa": 0.0}, @@ -775,26 +775,26 @@ async fn test_web_metrics_json() { "cognitive": {"sum": 0, "value": 0, "average": 0.0 , "min":0, "max":0}, "nargs": {"function_args": 0, "function_args_average": 0.0, "closure_args": 0, "closure_args_average": 0.0, "total": 0, "value": 0, "average": 0.0, "closure_args_max": 0, "closure_args_min": 0, "function_args_max": 0, "function_args_min": 0}, "nexits": {"sum": 0, "average": 0.0,"min":0,"max":0}, - "halstead": {"bugs": 0.000_942_552_557_372_941_4, - "difficulty": 1.0, - "effort": 4.754_887_502_163_468, - "length": 3, - "estimated_program_length": 2.0, - "purity_ratio": 0.666_666_666_666_666_6, - "level": 1.0, + "halstead": {"bugs": 0.002_712_967_490_108_627, + "difficulty": 2.0, + "effort": 23.219_280_948_873_62, + "length": 5, + "estimated_program_length": 8.0, + "purity_ratio": 1.6, + "level": 0.5, "total_operands": 1, - "total_operators": 2, - "vocabulary": 3, - "time": 0.264_160_416_786_859_36, + "total_operators": 4, + "vocabulary": 5, + "time": 1.289_960_052_715_201, "unique_operands": 1, - "unique_operators": 2, - "volume": 4.754_887_502_163_468}, + "unique_operators": 4, + "volume": 11.609_640_474_436_81}, "loc": {"cloc": 0, "ploc": 2, "lloc": 1, "sloc": 2, "blank": 0, "cloc_average": 0.0, "ploc_average": 2.0, "lloc_average": 1.0, "sloc_average": 2.0, "blank_average": 0.0, "cloc_min": 0, "ploc_min": 2, "lloc_min": 1, "sloc_min": 2, "blank_min": 0, "cloc_max": 0, "ploc_max": 2, "lloc_max": 1, "sloc_max": 2, "blank_max": 0}, "nom": {"functions": 1, "closures": 0, "functions_average": 1.0, "closures_average": 0.0, "total": 1, "average": 1.0, "closures_min": 0, "closures_max": 0, "functions_min": 1, "functions_max": 1}, "tokens": {"tokens": 6, "average": 6.0, "min": 6, "max": 6}, - "mi": {"original": 151.433_315_883_223_23, - "sei": 142.873_061_717_489_78, - "visual_studio": 88.557_494_668_551_6}, + "mi": {"original": 146.791_469_367_545_94, + "sei": 136.176_292_768_754_45, + "visual_studio": 85.842_964_542_424_53}, "abc": {"assignments": 0, "branches": 0, "conditions": 0, "magnitude": 0.0, "value": 0.0, "assignments_average": 0.0, "branches_average": 0.0, "conditions_average": 0.0, "assignments_min": 0, "assignments_max": 0, "branches_min": 0, "branches_max": 0, "conditions_min": 0, "conditions_max": 0}}, "name": "foo", "spaces": []}]} @@ -836,26 +836,26 @@ async fn test_web_metrics_json_unit() { "cognitive": {"sum": 0, "value": 0, "average": 0.0 , "min":0, "max":0}, "nargs": {"function_args": 0, "function_args_average": 0.0, "closure_args": 0, "closure_args_average": 0.0, "total": 0, "value": 0, "average": 0.0, "closure_args_max": 0, "closure_args_min": 0, "function_args_max": 0, "function_args_min": 0}, "nexits": {"sum": 0, "average": 0.0,"min":0,"max":0}, - "halstead": {"bugs": 0.000_942_552_557_372_941_4, - "difficulty": 1.0, - "effort": 4.754_887_502_163_468, - "length": 3, - "estimated_program_length": 2.0, - "purity_ratio": 0.666_666_666_666_666_6, - "level": 1.0, + "halstead": {"bugs": 0.002_712_967_490_108_627, + "difficulty": 2.0, + "effort": 23.219_280_948_873_62, + "length": 5, + "estimated_program_length": 8.0, + "purity_ratio": 1.6, + "level": 0.5, "total_operands": 1, - "total_operators": 2, - "vocabulary": 3, - "time": 0.264_160_416_786_859_36, + "total_operators": 4, + "vocabulary": 5, + "time": 1.289_960_052_715_201, "unique_operands": 1, - "unique_operators": 2, - "volume": 4.754_887_502_163_468}, + "unique_operators": 4, + "volume": 11.609_640_474_436_81}, "loc": {"cloc": 0, "ploc": 2, "lloc": 1, "sloc": 2, "blank": 0, "cloc_average": 0.0, "ploc_average": 1.0, "lloc_average": 0.5, "sloc_average": 1.0, "blank_average": 0.0, "cloc_min": 0, "ploc_min": 2, "lloc_min": 1, "sloc_min": 2, "blank_min": 0, "cloc_max": 0, "ploc_max": 2, "lloc_max": 1, "sloc_max": 2, "blank_max": 0}, "nom": {"functions": 1, "closures": 0, "functions_average": 0.5, "closures_average": 0.0, "total": 1, "average": 0.5, "closures_min": 0, "closures_max": 0, "functions_min": 0, "functions_max": 1}, "tokens": {"tokens": 6, "average": 3.0, "min": 0, "max": 6}, - "mi": {"original": 151.203_315_883_223_2, - "sei": 142.643_061_717_489_76, - "visual_studio": 88.422_991_744_574_97}, + "mi": {"original": 146.561_469_367_545_93, + "sei": 135.946_292_768_754_43, + "visual_studio": 85.708_461_618_447_91}, "abc": {"assignments": 0, "branches": 0, "conditions": 0, "magnitude": 0.0, "value": 0.0, "assignments_average": 0.0, "branches_average": 0.0, "conditions_average": 0.0, "assignments_min": 0, "assignments_max": 0, "branches_min": 0, "branches_max": 0, "conditions_min": 0, "conditions_max": 0}, "wmc": {"class_wmc_sum": 0, "interface_wmc_sum": 0, "total": 0}, "npm": {"class_npm_sum": 0, "interface_npm_sum": 0, "class_methods": 0, "interface_methods": 0, "class_coa": 0.0, "interface_coa": 0.0, "total": 0, "total_methods": 0, "coa": 0.0}, @@ -942,26 +942,26 @@ async fn test_web_metrics_plain() { "cognitive": {"sum": 0, "value": 0, "average": 0.0 , "min":0, "max":0}, "nargs": {"function_args": 0, "function_args_average": 0.0, "closure_args": 0, "closure_args_average": 0.0, "total": 0, "value": 0, "average": 0.0, "closure_args_max": 0, "closure_args_min": 0, "function_args_max": 0, "function_args_min": 0}, "nexits": {"sum": 0, "average": 0.0,"min":0,"max":0}, - "halstead": {"bugs": 0.000_942_552_557_372_941_4, - "difficulty": 1.0, - "effort": 4.754_887_502_163_468, - "length": 3, - "estimated_program_length": 2.0, - "purity_ratio": 0.666_666_666_666_666_6, - "level": 1.0, + "halstead": {"bugs": 0.002_712_967_490_108_627, + "difficulty": 2.0, + "effort": 23.219_280_948_873_62, + "length": 5, + "estimated_program_length": 8.0, + "purity_ratio": 1.6, + "level": 0.5, "total_operands": 1, - "total_operators": 2, - "vocabulary": 3, - "time": 0.264_160_416_786_859_36, + "total_operators": 4, + "vocabulary": 5, + "time": 1.289_960_052_715_201, "unique_operands": 1, - "unique_operators": 2, - "volume": 4.754_887_502_163_468}, + "unique_operators": 4, + "volume": 11.609_640_474_436_81}, "loc": {"cloc": 0, "ploc": 2, "lloc": 1, "sloc": 2, "blank": 0, "cloc_average": 0.0, "ploc_average": 1.0, "lloc_average": 0.5, "sloc_average": 1.0, "blank_average": 0.0, "cloc_min": 0, "ploc_min": 2, "lloc_min": 1, "sloc_min": 2, "blank_min": 0, "cloc_max": 0, "ploc_max": 2, "lloc_max": 1, "sloc_max": 2, "blank_max": 0}, "nom": {"functions": 1, "closures": 0, "functions_average": 0.5, "closures_average": 0.0, "total": 1, "average": 0.5, "closures_min": 0, "closures_max": 0, "functions_min": 0, "functions_max": 1}, "tokens": {"tokens": 6, "average": 3.0, "min": 0, "max": 6}, - "mi": {"original": 151.203_315_883_223_2, - "sei": 142.643_061_717_489_76, - "visual_studio": 88.422_991_744_574_97}, + "mi": {"original": 146.561_469_367_545_93, + "sei": 135.946_292_768_754_43, + "visual_studio": 85.708_461_618_447_91}, "abc": {"assignments": 0, "branches": 0, "conditions": 0, "magnitude": 0.0, "value": 0.0, "assignments_average": 0.0, "branches_average": 0.0, "conditions_average": 0.0, "assignments_min": 0, "assignments_max": 0, "branches_min": 0, "branches_max": 0, "conditions_min": 0, "conditions_max": 0}, "wmc": {"class_wmc_sum": 0, "interface_wmc_sum": 0, "total": 0}, "npm": {"class_npm_sum": 0, "interface_npm_sum": 0, "class_methods": 0, "interface_methods": 0, "class_coa": 0.0, "interface_coa": 0.0, "total": 0, "total_methods": 0, "coa": 0.0}, @@ -974,26 +974,26 @@ async fn test_web_metrics_plain() { "cognitive": {"sum": 0, "value": 0, "average": 0.0 , "min":0, "max":0}, "nargs": {"function_args": 0, "function_args_average": 0.0, "closure_args": 0, "closure_args_average": 0.0, "total": 0, "value": 0, "average": 0.0, "closure_args_max": 0, "closure_args_min": 0, "function_args_max": 0, "function_args_min": 0}, "nexits": {"sum": 0, "average": 0.0,"min":0,"max":0}, - "halstead": {"bugs": 0.000_942_552_557_372_941_4, - "difficulty": 1.0, - "effort": 4.754_887_502_163_468, - "length": 3, - "estimated_program_length": 2.0, - "purity_ratio": 0.666_666_666_666_666_6, - "level": 1.0, + "halstead": {"bugs": 0.002_712_967_490_108_627, + "difficulty": 2.0, + "effort": 23.219_280_948_873_62, + "length": 5, + "estimated_program_length": 8.0, + "purity_ratio": 1.6, + "level": 0.5, "total_operands": 1, - "total_operators": 2, - "vocabulary": 3, - "time": 0.264_160_416_786_859_36, + "total_operators": 4, + "vocabulary": 5, + "time": 1.289_960_052_715_201, "unique_operands": 1, - "unique_operators": 2, - "volume": 4.754_887_502_163_468}, + "unique_operators": 4, + "volume": 11.609_640_474_436_81}, "loc": {"cloc": 0, "ploc": 2, "lloc": 1, "sloc": 2, "blank": 0, "cloc_average": 0.0, "ploc_average": 2.0, "lloc_average": 1.0, "sloc_average": 2.0, "blank_average": 0.0, "cloc_min": 0, "ploc_min": 2, "lloc_min": 1, "sloc_min": 2, "blank_min": 0, "cloc_max": 0, "ploc_max": 2, "lloc_max": 1, "sloc_max": 2, "blank_max": 0}, "nom": {"functions": 1, "closures": 0, "functions_average": 1.0, "closures_average": 0.0, "total": 1, "average": 1.0, "closures_min": 0, "closures_max": 0, "functions_min": 1, "functions_max": 1}, "tokens": {"tokens": 6, "average": 6.0, "min": 6, "max": 6}, - "mi": {"original": 151.433_315_883_223_23, - "sei": 142.873_061_717_489_78, - "visual_studio": 88.557_494_668_551_6}, + "mi": {"original": 146.791_469_367_545_94, + "sei": 136.176_292_768_754_45, + "visual_studio": 85.842_964_542_424_53}, "abc": {"assignments": 0, "branches": 0, "conditions": 0, "magnitude": 0.0, "value": 0.0, "assignments_average": 0.0, "branches_average": 0.0, "conditions_average": 0.0, "assignments_min": 0, "assignments_max": 0, "branches_min": 0, "branches_max": 0, "conditions_min": 0, "conditions_max": 0}}, "name": "foo", "spaces": []}]} diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index 6b750ee2..50a7e088 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -154,6 +154,9 @@ Read it as follows. | `halstead/wide-distinct-fn` | Rust | width | per-child work at the space-merge boundary (#1106) | linear | | `abc/wide-let-declarators` | JavaScript | width | ABC's structural `const` check per `let` declarator (#1277) | linear | | `loc/wide-cfg-test-mod` | Rust | width | the `exclude_tests` prune arm's row bookkeeping (#1417) | linear | +| `halstead/perl-dash-keys` | Perl | width | the dash-key workaround's `=>` lookup per key (#1545) | linear | +| `halstead/kotlin-noinline-params` | Kotlin | width | the modifier recovery guard's next-token lookup (#1558) | linear | +| `abc/perl-pattern-list` | Perl | width | ABC's pattern-role climb through `( … )` (#1467) | linear | | `cyclomatic/cpp-plus-chain` | C++ | depth | shape control for the three rows below | linear | | `cyclomatic/cpp-and-chain` | C++ | depth | the `&&` / `||` applied-operator gate (#1533) | linear | | `cognitive/cpp-and-chain` | C++ | depth | the boolean-sequence arm on the same chain (#1533) | linear | diff --git a/docs/development/lessons_learned.md b/docs/development/lessons_learned.md index 8eb89ff9..f8a3e1e1 100644 --- a/docs/development/lessons_learned.md +++ b/docs/development/lessons_learned.md @@ -138,6 +138,7 @@ number and the higher number stays as a redirect. | [92](#92-an-optimizations-rationale-can-encode-the-waste-it-optimizes-for) | An optimization's rationale can encode the waste it optimizes for | | [93](#93-a-gate-that-reads-a-typed-accessor-is-invisible-to-the-front-ends-that-read-the-wire) | A gate that reads a typed accessor is invisible to the front-ends that read the wire | | [94](#94-a-sweep-that-finds-nothing-only-rules-out-what-its-predicate-can-express) | A sweep that finds nothing only rules out what its predicate can express | +| [95](#95-error-recovery-inserts-tokens-that-is_missing-does-not-flag) | Skip invented tokens by width, not `is_missing()` — valid code triggers it too | --- @@ -1481,6 +1482,14 @@ from TS's `is_string` (and TSX's `String3`) rather than adding it to is not a literal in either predicate, which is the resolution to reach for when the cross-walk turns one of these up. +**A token arm narrowed without its slot predicate** (#1540). The first +draft of the fix narrowed Perl ABC's `=~` arm so that a bound +substitution (`$x =~ s///`) no longer scored as a test by use. Because +`perl_condition_scores_itself` still listed `=~` as scoring itself, the +slot paid nothing, and `if ($x =~ s///)` fell from 1 to 0. The fix that +shipped gates both on one shared predicate. A perturbation of each side +fails a different subset of tests. + --- ## 36. `serde_json::to_value` re-sorts JSON object keys via `BTreeMap` @@ -3881,3 +3890,40 @@ never whether it is compiled wherever a caller is, though the derivation already listed `go` among the helper's needs (#1528). --- + +## 95. Error recovery inserts tokens that `is_missing()` does not flag + +**Lesson:** A guard meant to skip what the parser invented rather than +what the author wrote must key on **width** (`start_byte() == +end_byte()`), not on `Node::is_missing()`. Prove it with one malformed +fixture per grammar the guard covers. Do not assume the trap needs +invalid input: grammars cannot parse every valid program, and they +recover from the gap the same way. + +When tree-sitter recovers from a parse error, it can insert a token the +source lacks, such as a `;`, a `)` or an identifier. The inserted token +has zero width. Most are flagged MISSING. Some grammars' recovery also +builds zero-width identifiers *without* that flag. A classifier treats +either kind as an ordinary leaf, so the result is a plausible count, +not an error. Fixtures written from code the grammar handles never +contain one. + +**An operand spelled `""` in every language** (#1546). + +- Every getter that classified identifiers billed an inserted identifier + as an empty-text operand. An inserted `;` or `)` became a second + operator. +- Most instances were *valid* C++ that tree-sitter-cpp 0.23.4 cannot + parse, such as a `typename` functional cast used as a statement, or the + templated destructor call `dfs_state->~DfsState()`. There were + roughly 9,900 inserted `;` across DeepSpeech. +- The obvious fix, skipping `is_missing()` nodes, cleared C++ but left C# + and Elixir billing `""`, because their recovery builds the identifier + unflagged. A sweep with one malformed fixture per grammar showed it. + `compute_halstead` now skips any zero-width node, since a zero-width + node spells no token. +- Width answers "was a token written", not "is the input malformed": + valid trees contain zero-width nodes too, such as an empty Bash heredoc + body. + +--- diff --git a/src/concurrent_files_tests.rs b/src/concurrent_files_tests.rs index 79ec4fd0..62d9cd8e 100644 --- a/src/concurrent_files_tests.rs +++ b/src/concurrent_files_tests.rs @@ -470,13 +470,20 @@ fn without_path_verification_dispatches_every_path() { /// only way to observe the difference from outside: dispatch itself now /// happens on this thread. /// -/// Asserted as an upper bound plus "more than one", because the pool is -/// work-stealing: with few files a fast consumer can drain the channel -/// before its peers wake, so the exact count is not deterministic. The -/// old `num_jobs - 1` behaviour is still excluded — at `num_jobs = 2` it -/// permitted exactly one thread, and this fixture reaches two. +/// The count is taken behind a rendezvous: each consumer, on its first +/// file, waits for the other before returning. Counting the threads +/// that happened to run the callback is a point-in-time read: under +/// scheduler pressure the second consumer can start after the first has +/// drained all 200 files, so it existed but was never seen (#1519). With +/// the first consumer parked, the second is guaranteed a file whenever +/// it starts. The wait is bounded so that the pre-#1114 single consumer +/// fails the "more than one" assertion below instead of hanging. #[test] fn num_jobs_is_the_consumer_count_not_a_budget_shared_with_a_producer() { + // Generous enough for any scheduler delay a loaded CI host imposes; + // it is only ever waited out in full when a consumer is missing. + const RENDEZVOUS_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + let tmp = Builder::new().prefix("jobs").tempdir().expect("tempdir"); let root = tmp.path(); let mut paths = Vec::new(); @@ -488,14 +495,24 @@ fn num_jobs_is_the_consumer_count_not_a_budget_shared_with_a_producer() { let threads = Arc::new(Mutex::new(std::collections::HashSet::new())); let sink = Arc::clone(&threads); + let gate = (Mutex::new(0_usize), std::sync::Condvar::new()); let caller = thread::current().id(); ConcurrentRunner::new(2, move |_path: PathBuf, _cfg: &()| { - sink.lock() + let first_file = sink + .lock() .expect("uncontended in test") .insert(thread::current().id()); - // Hold the worker briefly so the second consumer is given a - // chance to pick up work rather than losing every race. - thread::sleep(std::time::Duration::from_micros(50)); + if first_file { + let (count, ready) = &gate; + let mut count = count.lock().expect("uncontended in test"); + *count += 1; + ready.notify_all(); + drop( + ready + .wait_timeout_while(count, RENDEZVOUS_TIMEOUT, |n| *n < 2) + .expect("uncontended in test"), + ); + } Ok(()) }) .run((), FilesData { paths }) diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index de0c449d..f16f646c 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -80,7 +80,7 @@ mod tcl; /// Java `default:` precedent). The short- /// circuit logical operators `&&` and `||` (and per-language /// equivalents — Python's `and` / `or`, Lua's `and` / `or`, Tcl's -/// `&&` / `||`, Perl's `&&` / `||` / `//` / `and` / `or` / `xor`) +/// `&&` / `||`, Perl's `&&` / `||` / `//` / `and` / `or`) /// are deliberately **not** counted on their own. The paper's /// worked Listing 2 annotates `(am >= 0 && am <= 0xF) ? '/' : 'C'` /// as `accc` — three conditions for `>=`, `<=`, `?`, zero for @@ -527,6 +527,32 @@ pub(super) fn count_negated_operand<'a>( } } +// A language's scorer for one operand: its `count_boolean_slot` or +// `count_negated_operand` wrapper. +pub(super) type OperandScorer = fn(&Node, &mut f64); + +// Fitzpatrick Rule 9 for a chain whose grammar names its two operands +// `left` and `right`: each is scored by `score`. Read by field rather +// than as every operand child, so a comment beside the operator — a +// named child too — does not pay. The caller owns the gate deciding +// that `chain` is a chain at all. +pub(super) fn count_field_operands(chain: &Node, score: OperandScorer, conditions: &mut f64) { + for field in ["left", "right"] { + if let Some(operand) = chain.child_by_field_name(field) { + score(&operand, conditions); + } + } +} + +// Each operand child of `list` — an argument list, a `return` value +// list, a chain whose grammar names no operand field — scored by +// `score`. `is_operand` keeps punctuation and comments out. +pub(super) fn count_each_operand(list: &Node, score: OperandScorer, conditions: &mut f64) { + for operand in list.children().filter(is_operand) { + score(&operand, conditions); + } +} + // Default no-op `Abc` impls. Audited in #188; the matrix below // records the rationale for every entry so the no-op default is a // deliberate choice, not scaffolding leftover. @@ -591,7 +617,9 @@ mod tests { #[cfg(any( feature = "c", feature = "cpp", + feature = "csharp", feature = "go", + feature = "java", feature = "javascript", feature = "mozcpp", feature = "mozjs", @@ -3280,10 +3308,85 @@ mod tests { void F(bool a, bool b, bool c) {} }", "foo.cs", - |metric| insta::assert_json_snapshot!(metric.abc), + |metric| { + // expected: `==` +1, `<` +1, and the negated argument + // `!x.Equals(y)` +1 — it scored 0 until #1537, when every + // C# argument reached the peel still in its `argument` + // wrapper. + assert_eq!(metric.abc.conditions_sum(), 3); + insta::assert_json_snapshot!(metric.abc); + }, ); } + // #1537: every C# argument is wrapped in an `argument` node that the + // negation peel declined, so a negated argument scored 0 where every + // other C-family language scores 1. Each row is a C# call next to its + // Java twin, which takes the expected value from outside the fix. + #[cfg(all(feature = "csharp", feature = "java"))] + #[test] + fn csharp_negated_call_argument_matches_java() { + let rows = [ + ("G(!a);", "g(!a);", 1), + ("G((!a));", "g((!a));", 1), + ("G(!a, !b);", "g(!a, !b);", 2), + ("G(a, !b);", "g(a, !b);", 1), + ("new B(!a);", "new B(!a);", 1), + ("G(-a);", "g(-a);", 0), + // A negated comparison pays once, for its `>`; a chain pays + // per operand. Neither argument may pay again as a negation. + ("G(!(a > b));", "g(!(a > b));", 1), + ("G(!a && b);", "g(!a && b);", 2), + ]; + for (csharp, java, expected) in rows { + let cs = abc_conditions( + LANG::Csharp, + &format!("class A {{ void M() {{ {csharp} }} }}"), + ); + let jv = abc_conditions(LANG::Java, &format!("class A {{ void m() {{ {java} }} }}")); + assert_eq!(jv, expected, "Java twin `{java}`"); + assert_eq!(cs, expected, "C# `{csharp}` against Java `{java}`"); + } + } + + // The C#-only spellings around an argument. The value is the + // `argument` wrapper's last operand, so a named-argument label, a + // passing-mode keyword or a comment before it must not hide it. + #[cfg(feature = "csharp")] + #[test] + fn csharp_negated_call_argument_spellings() { + let rows = [ + // A named argument: the value follows `name` and `:`. + ("class A { void M() { G(name: !a); } }", 1), + // A comment between the label and the value sits inside the + // `argument` wrapper, so a positional read of the value lands + // on it. A comment before or after a whole argument is a + // sibling of the wrapper instead and never reaches this read. + ("class A { void M() { G(name: /*c*/ !a); } }", 1), + // `ref` / `out` / `in` precede a variable, never a negation. A + // control only: 0 is also what the pre-#1537 walk scored, so + // this row cannot fail for any defect the fix addresses. + ("class A { void M() { G(ref x, out var y, in z); } }", 0), + // The lambda arm pays for its `!a` body; the argument holding + // the lambda is not a negation and must not pay again. + ("class A { void M() { G(x => !a); } }", 1), + // `: base(!a)` — a `constructor_initializer`'s list reaches the + // walk like any call's. + ("class A : B { A(bool a) : base(!a) {} }", 1), + // The two primary-constructor base calls: the record nests its + // list in a `primary_constructor_base_type`, the class's list + // is the branch node itself and is scored on the branch path. + ("record R(bool a) : Base(!a);", 1), + ("class S(bool a) : Base(!a) {}", 1), + // An indexer's `bracketed_argument_list` is not a call, as + // Java's array index is not. + ("class A { void M() { var q = o[!a]; } }", 0), + ]; + for (src, expected) in rows { + assert_eq!(abc_conditions(LANG::Csharp, src), expected, "{src}"); + } + } + #[cfg(feature = "csharp")] #[test] fn csharp_if_single_conditions() { @@ -6091,9 +6194,9 @@ function f(int $a, int $b): int { fn php_low_precedence_keyword_logical_ops_trigger_walker() { // Regression: pre-fix, `$a or $b` reported 0 conditions // because the dispatcher only handled `AMPAMP|PIPEPIPE`, - // skipping the PHP-specific `and` / `or` / `xor` keyword - // forms even though they parse under the same - // `binary_expression` shape. + // skipping the PHP-specific `and` / `or` keyword forms even + // though they parse under the same `binary_expression` shape. + // (`xor` is no chain: it is a value since #1536.) check_metrics::( "( "sub f {\n\ my $r;\n\ @@ -13983,9 +14135,9 @@ end // 7 `=` tokens (one per reassignment line). assert_eq!(metric.abc.assignments_sum(), 7); assert_eq!(metric.abc.branches_sum(), 0); - // 6 chain lines × 2 operands + 1 ternary node + 1 for - // its bare `$a` condition operand = 14. - assert_eq!(metric.abc.conditions_sum(), 14); + // 5 chain lines × 2 operands + 1 ternary node + 1 for + // its bare `$a` condition operand = 12. + assert_eq!(metric.abc.conditions_sum(), 12); insta::assert_json_snapshot!(metric.abc); }, ); @@ -14027,6 +14179,92 @@ end ); } + // Issue #1273. tree-sitter-perl 1.1.2 parses `$a > $b ? 1 : 2` as + // `$a > ($b ? 1 : 2)`, and the ternary's condition slot then paid + // for `$b`: 3 against JavaScript's 2 for `a > b ? 1 : 2`. Every + // chaining comparison does it; each row must also equal its + // parenthesised spelling, which parses correctly and so takes its + // value from outside the fix. + #[cfg(feature = "perl")] + #[test] + fn perl_comparison_conditioned_ternary_counts_the_comparison_once() { + let operators = [ + "<", ">", "<=", ">=", "lt", "gt", "le", "ge", "==", "!=", "eq", "ne", + ]; + for op in operators { + let bare = format!("sub f {{ my $m = $a {op} $b ? 1 : 2; }}\n"); + let grouped = format!("sub f {{ my $m = ($a {op} $b) ? 1 : 2; }}\n"); + // The ternary (1) and the comparison token (1). + assert_eq!(abc_conditions(LANG::Perl, &bare), 2, "`{op}`"); + assert_eq!(abc_conditions(LANG::Perl, &grouped), 2, "`({op})`"); + } + } + + // Issue #1273, the general shape: a chaining comparison swallows + // every lower-precedence operator to its right, so a logical chain + // misparses too — `$a > $b && $c` is `$a > ($b && $c)`, and the + // chain walker paid for `$b`. Expected values are the JavaScript + // twins (`a > b && c ? 1 : 2` = 3, …) or, where JavaScript has no + // spelling, the parenthesised Perl one. + #[cfg(feature = "perl")] + #[test] + fn perl_misparsed_comparison_operand_is_not_a_condition() { + let cases = [ + // `<` `<` `?`; `($a < $b < $c) ? 1 : 2` is 3. + ("my $m = $a < $b < $c ? 1 : 2;", 3), + // `>` `?` `>` `?`, as `a > b ? (c > d ? 1 : 2) : 3`. + ("my $m = $a > $b ? $c > $d ? 1 : 2 : 3;", 4), + // Two ternaries and two comparisons, as the JS twin. + ("if ($a > $b ? $c : $d) { } return $a == $b ? 1 : 0;", 4), + // The chain is the ternary's condition: `>` `?` `$c`. + ("my $m = $a > $b && $c ? 1 : 2;", 3), + // The ternary hangs off the chain's right comparison. + ("my $m = $x > 0 && $y > 0 ? 1 : 2;", 3), + ("my $m = $x && $a > $b ? 1 : 2;", 3), + ("if ($a > $b && $c) { }", 2), + ("if ($a eq $b && $c eq $d) { }", 2), + // `&&` is the first operand of `||`: the climb crosses it. + ("if ($a > $b && $c || $d) { }", 3), + ("if ($a > $b && $c && $d) { }", 3), + // `and` is a two-operand `unary_expression`, which the + // climb crosses too: `(($a > $b) && $c) and $d` is 3. + ("if ($a > $b and $c) { }", 2), + ("if ($a > $b && $c and $d) { }", 3), + // A comment between the operator and the ternary is an + // `extra` sibling, not the operator. + ("my $m = $a > # c\n $b ? 1 : 2;", 2), + ]; + for (body, expected) in cases { + let src = format!("sub f {{ {body} }}\n"); + assert_eq!(abc_conditions(LANG::Perl, &src), expected, "`{body}`"); + } + } + + // Issue #1273: shapes the workaround must leave alone. The explicit + // `$a > ($b ? 1 : 2)` puts an `array` between the comparison and + // the ternary, so `$b` really is a condition: 3, as + // `a > (b ? 1 : 2)` in JavaScript. `<=>`, `cmp` and `=~` do not + // chain and parse correctly; those rows pin that grammar fact, not + // the workaround's operator list — adding `<=>` to the list moves + // nothing, since no tree ever puts a ternary straight after it. + #[cfg(feature = "perl")] + #[test] + fn perl_correctly_parsed_ternaries_keep_their_condition_slot() { + let cases = [ + ("my $m = ($a > $b) ? 1 : 2;", 2), + ("my $m = $a <=> $b ? 1 : 2;", 2), + ("my $m = $a cmp $b ? 1 : 2;", 2), + ("my $m = $a =~ /x/ ? 1 : 2;", 2), + ("my $m = $a > ($b ? 1 : 2);", 3), + ("my $m = $a ? !$b : !$c;", 4), + ("my $m = $a && $b ? 1 : 2;", 3), + ]; + for (body, expected) in cases { + let src = format!("sub f {{ {body} }}\n"); + assert_eq!(abc_conditions(LANG::Perl, &src), expected, "`{body}`"); + } + } + #[cfg(feature = "perl")] #[test] fn perl_elsif_and_else_count_conditions() { @@ -14122,7 +14360,7 @@ end #[test] fn perl_if_multiple_conditions() { // Fitzpatrick Rule 9 walker (issue #403): each operand of a - // `&&` / `||` / `//` / `and` / `or` / `xor` chain is one + // `&&` / `||` / `//` / `and` / `or` chain is one // condition. ScalarVariable operands ($a, $b, …) qualify as // terminal-bool kinds for the walker. check_metrics::( @@ -14347,8 +14585,8 @@ end // reads what it read before. `bound_out` / `nbound` pin the gate on // `=~` / `!~` — the operator token scores, the pattern under it does // not — and `ctrl_out` is the `==` control the rest level with. - // `s///`, `tr///` and `qr//` are not matches and stay at 0; whether - // the first two should score is #1475. + // `s///`, `tr///` and `qr//` are not matches and stay at 0; the + // bound `s///` / `tr///` agree (#1540, the test below). #[cfg(feature = "perl")] #[test] fn perl_bare_match_is_a_condition_by_use() { @@ -14408,6 +14646,81 @@ end ); } + // A substitution or transliteration bound with `=~` / `!~` is the + // same edit as its bare form on `$_`, so it scores by use exactly as + // that twin does: 0 (#1540). Each `*_s` / `*_tr` / `*_y` row below + // was 1 through the operator token before the gate; `count` (an + // assigned count) and `sr` (a `/r` copy, a value) are uses too. A + // match keeps its 1 whatever spells it — a pattern, `qr//`, or a + // variable holding one. The `if_*` rows are the slot side: each pays + // once, as its bare twin `if_s` does, because + // `perl_condition_scores_itself` stops claiming a rewrite binding in + // the same change; `tern_s` and `neg_s` reach that predicate through + // a ternary condition and a `!`-proved `return` operand. + #[cfg(feature = "perl")] + #[test] + fn perl_bound_rewrite_scores_like_its_bare_twin() { + check_func_space::( + "sub ctrl { my ($x) = @_; $x; return $x; }\n\ + sub m_bare { my ($x) = @_; /re/; return $x; }\n\ + sub m_bound { my ($x) = @_; $x =~ /re/; return $x; }\n\ + sub m_nbound { my ($x) = @_; $x !~ /re/; return $x; }\n\ + sub m_qr { my ($x) = @_; $x =~ qr/re/; return $x; }\n\ + sub m_var { my ($x, $q) = @_; $x =~ $q; return $x; }\n\ + sub s_bare { my ($x) = @_; s/a/b/; return $x; }\n\ + sub s_bound { my ($x) = @_; $x =~ s/a/b/; return $x; }\n\ + sub s_nbound { my ($x) = @_; $x !~ s/a/b/; return $x; }\n\ + sub tr_bare { my ($x) = @_; tr/a/b/; return $x; }\n\ + sub tr_bound { my ($x) = @_; $x =~ tr/a/b/; return $x; }\n\ + sub tr_nbound { my ($x) = @_; $x !~ tr/a/b/; return $x; }\n\ + sub y_bound { my ($x) = @_; $x =~ y/a/b/; return $x; }\n\ + sub count { my ($x) = @_; my $n = ($x =~ s/a/b/g); return $x; }\n\ + sub sr { my ($x) = @_; my $r = $x =~ s/a/b/r; return $r; }\n\ + sub if_s { my ($x) = @_; if (s/a/b/) { g(); } return $x; }\n\ + sub if_m { my ($x) = @_; if ($x =~ /re/) { g(); } return $x; }\n\ + sub if_bs { my ($x) = @_; if ($x =~ s/a/b/) { g(); } return $x; }\n\ + sub if_nbs { my ($x) = @_; if ($x !~ s/a/b/) { g(); } return $x; }\n\ + sub if_btr { my ($x) = @_; if ($x =~ tr/a/b/) { g(); } return $x; }\n\ + sub if_by { my ($x) = @_; if ($x =~ y/a/b/) { g(); } return $x; }\n\ + sub if_bsr { my ($x) = @_; if ($x =~ s/a/b/r) { g(); } return $x; }\n\ + sub tern_s { my ($x) = @_; return ($x =~ s/a/b/) ? 1 : 0; }\n\ + sub neg_s { my ($x) = @_; return !($x =~ s/a/b/); }\n", + "foo.pl", + |space| { + assert_members_score( + &space, + &[ + ("ctrl", 0, 1), + ("m_bare", 1, 1), + ("m_bound", 1, 1), + ("m_nbound", 1, 1), + ("m_qr", 1, 1), + ("m_var", 1, 1), + ("s_bare", 0, 1), + ("s_bound", 0, 1), + ("s_nbound", 0, 1), + ("tr_bare", 0, 1), + ("tr_bound", 0, 1), + ("tr_nbound", 0, 1), + ("y_bound", 0, 1), + ("count", 0, 1), + ("sr", 0, 1), + ("if_s", 1, 2), + ("if_m", 1, 2), + ("if_bs", 1, 2), + ("if_nbs", 1, 2), + ("if_btr", 1, 2), + ("if_by", 1, 2), + ("if_bsr", 1, 2), + // The ternary node plus its condition slot. + ("tern_s", 2, 2), + ("neg_s", 1, 1), + ], + ); + }, + ); + } + // The pattern `split` takes first is its delimiter, not a match // against `$_`, so it is no condition (#1467) in any spelling: the // spaced and parenthesised call forms, the `m{}` pattern, and the @@ -14477,6 +14790,110 @@ end }); } + // tree-sitter-perl names its separators, so a trailing `,` or `=>` + // was read as the last element of a condition list and paid a slot + // of its own: `if ($a > 1,)` scored 2. `perl -MO=Deparse` reduces + // every `*_sep` row to its `*_twin`, which is where the expected + // values come from; the chain rows put the list inside a `&&` + // operand, which reaches the same peel through + // `perl_count_chain_operands`. + #[cfg(feature = "perl")] + #[test] + fn perl_trailing_separator_scores_like_its_bare_twin() { + check_func_space::( + "sub cmp_twin { my ($a) = @_; if ($a > 1) { g(); } }\n\ + sub cmp_comma { my ($a) = @_; if ($a > 1,) { g(); } }\n\ + sub cmp_fat { my ($a) = @_; if ($a > 1 =>) { g(); } }\n\ + sub var_twin { my ($a) = @_; if ($a) { g(); } }\n\ + sub var_comma { my ($a) = @_; if ($a,) { g(); } }\n\ + sub chain_twin { my ($a, $b) = @_; if (($a > 1) && $b) { g(); } }\n\ + sub chain_comma { my ($a, $b) = @_; if (($a > 1,) && $b) { g(); } }\n", + "foo.pl", + |space| { + assert_members_score( + &space, + &[ + ("cmp_twin", 1, 2), + ("cmp_comma", 1, 2), + ("cmp_fat", 1, 2), + ("var_twin", 1, 2), + ("var_comma", 1, 2), + ("chain_twin", 2, 3), + ("chain_comma", 2, 3), + ], + ); + }, + ); + } + + // A pattern's role is read on the outermost `( … )` around it, which + // is the value inside it, so each `*_paren` row scores as its bare + // `*_twin` (#1467, #1540); `perl -MO=Deparse` prints each pair + // identically. The reversed binding is the other direction: in + // `/foo/ =~ $x` the pattern is the *left* operand, so it first + // matches `$_` and that result is bound (Deparse: `/foo/ =~ /$x/`) — + // the match and the `=~` both pay, as in the explicit `rev_twin`. + #[cfg(feature = "perl")] + #[test] + fn perl_pattern_role_reads_through_parentheses() { + check_func_space::( + "sub bind_twin { my ($x) = @_; return $x =~ /foo/; }\n\ + sub bind_paren { my ($x) = @_; return $x =~ (/foo/); }\n\ + sub bind_parens_m { my ($x) = @_; return $x =~ (((m{foo}))); }\n\ + sub bind_list { my ($x, $y) = @_; return $x =~ (/foo/, $y); }\n\ + sub bind_list_twin { my ($x, $y) = @_; /foo/; return $x =~ $y; }\n\ + sub split_twin { my ($x) = @_; my @p = split(/foo/, $x); return @p; }\n\ + sub split_paren { my ($x) = @_; my @p = split((/foo/), $x); return @p; }\n\ + sub split_sole { my @p = split(/foo/); return @p; }\n\ + sub subst_twin { my ($x) = @_; $x =~ s/a/b/; return $x; }\n\ + sub subst_paren { my ($x) = @_; $x =~ (s/a/b/); return $x; }\n\ + sub rev { my ($x) = @_; return /foo/ =~ $x; }\n\ + sub rev_twin { my ($x) = @_; my $m = /foo/; return $m =~ $x; }\n", + "foo.pl", + |space| { + assert_members_score( + &space, + &[ + ("bind_twin", 1, 1), + ("bind_paren", 1, 1), + ("bind_parens_m", 1, 1), + // A comma operator: `/foo/` matches `$_` and `$y` + // is what binds, so the match pays as in the twin. + ("bind_list", 2, 1), + ("bind_list_twin", 2, 1), + ("split_twin", 0, 1), + ("split_paren", 0, 1), + // The pattern is the whole argument list, which is + // the call's own and no wrapper to climb through. + ("split_sole", 0, 1), + ("subst_twin", 0, 1), + ("subst_paren", 0, 1), + ("rev", 2, 1), + ("rev_twin", 2, 1), + ], + ); + }, + ); + } + + // Only the builtin `split` takes its first pattern as a delimiter: + // `My::split` is a user sub, so the pattern it is handed is an + // ordinary match, as it is for `My::other` (#1467). `CORE::split` + // is the builtin by its full name. + #[cfg(feature = "perl")] + #[test] + fn perl_split_exemption_is_the_builtin_only() { + check_func_space::( + "sub mine { my ($x) = @_; my @p = My::split(/foo/, $x); return @p; }\n\ + sub other { my ($x) = @_; my @p = My::other(/foo/, $x); return @p; }\n\ + sub core { my ($x) = @_; my @p = CORE::split(/foo/, $x); return @p; }\n", + "foo.pl", + |space| { + assert_members_score(&space, &[("mine", 1, 1), ("other", 1, 1), ("core", 0, 1)]); + }, + ); + } + // ---------- Lua ABC tests ---------- #[cfg(feature = "lua")] @@ -18007,8 +18424,13 @@ mod perl_statement_modifier_parity { /// C#, Java, Kotlin, Rust and Go name no literal kind at all and stay /// that way: a bare literal in a boolean slot is a compile error there, /// so there is nothing to count. The C family is integer-truthy, and -/// its slots have paid for a `string_literal` since #1526 made every -/// slot pay for an occupant no other arm counts. +/// #1462 deferred it for its corpus exposure. #1526 then closed its gap +/// without naming a kind, by making every slot pay for an occupant no +/// other arm counts; its rows here (#1469) pin that answer, because a +/// kind `cpp_condition_scores_itself` wrongly claimed would drop a slot +/// exactly as the missing terminal kind once did. Its `true`, `false` +/// and `null` rows are the exception to "measured short": the former +/// set already named them, and they stay so the pin covers every kind. /// /// Three things each row pins that a conditions comparison alone /// cannot: @@ -18046,7 +18468,11 @@ mod perl_statement_modifier_parity { feature = "groovy", feature = "perl", feature = "ruby", - feature = "elixir" + feature = "elixir", + feature = "c", + feature = "cpp", + feature = "mozcpp", + feature = "objc" ))] mod literal_bool_operands { use crate::test_support::{assert_fixture_spells, metrics_verbatim}; @@ -18094,6 +18520,29 @@ mod literal_bool_operands { ("function f() {\n if ({}) { return 1; }\n}\n", 1, 3), ]; + /// The five non-numeric literal kinds every C-family grammar shares, + /// then the grammar's own. The string and null spellings are passed + /// in because Objective-C writes its strings `@"s"` and C++ its null + /// `nullptr`; each still parses to the shared `string_literal` / + /// `null` kind, so the row stays one spelling per kind. + macro_rules! c_family_literals { + ($Lang:ident, $string:literal, $null:literal $(, $extra:expr)* $(,)?) => { + &[ + ($string, crate::$Lang::StringLiteral as u16), + ("\"a\" \"b\"", crate::$Lang::ConcatenatedString as u16), + ("true", crate::$Lang::True as u16), + ("false", crate::$Lang::False as u16), + ($null, crate::$Lang::Null as u16), + $($extra,)* + ] + }; + } + + const C_FAMILY_SLOTS: [Slot; 2] = [ + ("int f(int a) {\n return a && {};\n}\n", 2, 3), + ("int f() {\n if ({}) { return 1; }\n}\n", 1, 3), + ]; + fn conditions(lang: LANG, source: &str) -> u64 { metrics_verbatim(lang, source.as_bytes(), MetricsOptions::default()) .abc @@ -18140,6 +18589,14 @@ mod literal_bool_operands { LANG::Ruby => assert_fixture_spells::(source, "f.rb", kinds), #[cfg(feature = "elixir")] LANG::Elixir => assert_fixture_spells::(source, "f.ex", kinds), + #[cfg(feature = "c")] + LANG::C => assert_fixture_spells::(source, "f.c", kinds), + #[cfg(feature = "cpp")] + LANG::Cpp => assert_fixture_spells::(source, "f.cpp", kinds), + #[cfg(feature = "mozcpp")] + LANG::Mozcpp => assert_fixture_spells::(source, "f.cpp", kinds), + #[cfg(feature = "objc")] + LANG::Objc => assert_fixture_spells::(source, "f.m", kinds), other => panic!("{other:?} has a case row but no parser arm"), } } @@ -18188,16 +18645,26 @@ mod literal_bool_operands { /// `quoted_atom` — a separate production from `atom`, so /// `:"q a"` scored zero while `:atom` scored one — and the four /// collection literals `list` / `tuple` / `map` / `bitstring`. + /// - C family: `string_literal` (one kind for `"s"`, `L"s"`, `u8"s"` + /// and Objective-C's `@"s"`), `concatenated_string`, `true`, + /// `false` and `null` (`NULL`, and C++'s `nullptr`, which the + /// grammar wraps in a `null` node). C++ adds `raw_string_literal` + /// and `user_defined_literal` (`"s"_x`, and `1.0_km` around a + /// number); Objective-C adds its boxed forms — `array_literal`, + /// `dictionary_literal`, `at_expression` (`@1`, `@YES`, `@(b)`, + /// `@protocol(P)`), `selector_expression` and `encode_expression`. + /// `char_literal` is numeric and pinned in the sibling module. + /// Objective-C's `nil`, `YES` and `NO` parse as plain identifiers, + /// so they need no row of their own. /// - /// Shapes that measured short and are deliberately absent, none of - /// them a literal: JavaScript's `this` and Groovy's + /// Shapes that measured short and are not rows here, none of them a + /// literal: JavaScript's `this` and Groovy's /// `object_creation_expression` (recorded in #1462); Perl's /// `s///` and `tr///` (operations on `$_` evaluating to a count), - /// `anonymous_function` and `array_dereference`; and Ruby's - /// `lambda`. Elixir's `anonymous_function` and `&f/1` capture were - /// on this list until #1527 moved its slots off a terminal-kind set, - /// after which every operand scores unless an operator already - /// counts it. + /// `anonymous_function` and `array_dereference`; Ruby's `lambda`; + /// and Elixir's `anonymous_function` and `&f/1` capture. Every one + /// pays its slot since #1526 (Elixir's since #1527), and + /// `slot_pays_tests` pins them against an identifier control (#1475). fn cases(lang: LANG) -> Option { Some(match lang { LANG::Javascript => (JS_SLOTS, "b", js_literals!(Javascript, String2), 7), @@ -18342,6 +18809,51 @@ mod literal_bool_operands { ], 8, ), + LANG::C => ( + C_FAMILY_SLOTS, + "b", + c_family_literals!(C, "\"s\"", "NULL"), + 5, + ), + LANG::Cpp => ( + C_FAMILY_SLOTS, + "b", + c_family_literals!( + Cpp, + "\"s\"", + "nullptr", + ("R\"(x)\"", crate::Cpp::RawStringLiteral as u16), + ("\"s\"_x", crate::Cpp::UserDefinedLiteral as u16), + ), + 7, + ), + LANG::Mozcpp => ( + C_FAMILY_SLOTS, + "b", + c_family_literals!( + Mozcpp, + "\"s\"", + "nullptr", + ("R\"(x)\"", crate::Mozcpp::RawStringLiteral as u16), + ("\"s\"_x", crate::Mozcpp::UserDefinedLiteral as u16), + ), + 7, + ), + LANG::Objc => ( + C_FAMILY_SLOTS, + "b", + c_family_literals!( + Objc, + "@\"s\"", + "NULL", + ("@[]", crate::Objc::ArrayLiteral as u16), + ("@{}", crate::Objc::DictionaryLiteral as u16), + ("@1", crate::Objc::AtExpression as u16), + ("@selector(x)", crate::Objc::SelectorExpression as u16), + ("@encode(int)", crate::Objc::EncodeExpression as u16), + ), + 10, + ), _ => return None, }) } diff --git a/src/metrics/abc/bash.rs b/src/metrics/abc/bash.rs index d99e1079..9ef6d487 100644 --- a/src/metrics/abc/bash.rs +++ b/src/metrics/abc/bash.rs @@ -10,6 +10,7 @@ )] use super::{Abc, Stats}; +use crate::lang_helpers::bash::{bash_test_connective, bash_test_eq_count}; use crate::*; impl Abc for BashCode { @@ -36,8 +37,11 @@ impl Abc for BashCode { // Three condition signals share this arm: // // - Comparison operators inside `[[ … ]]` and `(( … ))`, plus - // the prefix test operators `-z`, `-n`, `-eq`, `-lt`, … which - // the grammar emits as `Bash::TestOperator`. + // the test operators `-z`, `-n`, `-eq`, `-lt`, … which the + // grammar emits as `Bash::TestOperator`. The `-a` / `-o` + // connectives share that kind and are excluded: they join + // two tests the way `&&` / `||` do, and neither pays + // (#1536). // - Control-flow branches (`if`/`elif`/`while`). A Bash predicate // is a command, so the branch keyword itself is the only // condition signal. These branch keywords mirror the matching @@ -53,7 +57,6 @@ impl Abc for BashCode { | Bash::LTEQ | Bash::GTEQ | Bash::EQTILDE - | Bash::TestOperator | Bash::IfStatement | Bash::ElifClause | Bash::WhileStatement @@ -61,6 +64,19 @@ impl Abc for BashCode { | Bash::TernaryExpression2 => { stats.conditions += 1.; } + Bash::TestOperator + if ancestors + .parent(node) + .is_none_or(|parent| bash_test_connective(node, &parent, code).is_none()) => + { + stats.conditions += 1.; + } + // `=` compares strings inside `[ … ]` / `[[ … ]]`, exactly as + // `==` does, and assigns inside `(( … ))` (#1536). Each test + // counts its own, once, from the top. + Bash::TestCommand => { + stats.conditions += bash_test_eq_count(node) as f64; + } // `<` and `>` are comparisons only inside a `binary_expression`. // The same two tokens spell an I/O redirection (`cmd > out`, // `read x < in`), which the grammar parents under diff --git a/src/metrics/abc/cpp.rs b/src/metrics/abc/cpp.rs index 56ff03e8..6753adf3 100644 --- a/src/metrics/abc/cpp.rs +++ b/src/metrics/abc/cpp.rs @@ -9,7 +9,10 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, wrapped_operand}; +use super::{ + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, is_operand, wrapped_operand, +}; use crate::metrics::cyclomatic::cpp_operator_is_applied; use crate::*; @@ -142,9 +145,7 @@ pub(super) fn cpp_count_return(statement: &Node, conditions: &mut f64) { // Each operand of a call's `argument_list` or of an Objective-C message // is a negated operand (`f(!a, !b)`). pub(super) fn cpp_count_arguments(list: &Node, conditions: &mut f64) { - for operand in list.children().filter(is_operand) { - cpp_count_negated(&operand, conditions); - } + count_each_operand(list, cpp_count_negated, conditions); } // Fitzpatrick Rule 9 (C++ in Figure 3, #403): each operand of an `&&` / @@ -167,12 +168,7 @@ pub(super) fn cpp_count_arguments(list: &Node, conditions: &mut f64) { // arms do, scores nothing. pub(super) fn cpp_count_chain_operands(chain: &Node, conditions: &mut f64) { match chain.kind() { - "binary_expression" => { - let operands = ["left", "right"].map(|field| chain.child_by_field_name(field)); - for operand in operands.iter().flatten() { - cpp_count_condition(operand, conditions); - } - } + "binary_expression" => count_field_operands(chain, cpp_count_condition, conditions), "fold_expression" => { let mut written = 0; for operand in chain.children().filter(is_operand) { @@ -373,7 +369,11 @@ impl Abc for CppCode { IfStatement | WhileStatement | DoStatement => { cpp_count_condition_slot(node, &mut stats.conditions); } - ReturnStatement => { + // `co_return` is a coroutine's `return` (#1547), so its value + // is read the same way: `co_return !x` counts one. `co_yield` + // hands back a value without returning, as the JS family's + // `yield` does, and like it scores no value slot. + ReturnStatement | CoReturnStatement => { cpp_count_return(node, &mut stats.conditions); } // `f(!a, !b)` — argument list walker. Two aliases — diff --git a/src/metrics/abc/csharp.rs b/src/metrics/abc/csharp.rs index bde8c98f..5ceee2fb 100644 --- a/src/metrics/abc/csharp.rs +++ b/src/metrics/abc/csharp.rs @@ -10,8 +10,8 @@ )] use super::{ - Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, last_operand, peel, - wrapped_operand, + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, last_operand, peel, wrapped_operand, }; use crate::macros::{csharp_paren_expr_kinds, csharp_prefix_unary_expr_kinds}; use crate::*; @@ -166,21 +166,20 @@ fn csharp_count_negated_slot(slot: Option, conditions: &mut f64) { // Each argument of a call is a negated operand. fn csharp_count_arguments(arguments: &Node, conditions: &mut f64) { - for argument in arguments.children().filter(is_operand) { - csharp_count_negated(&argument, conditions); - } + count_each_operand(arguments, csharp_count_argument, conditions); } -// Fitzpatrick Rule 9: each operand of an `&&` / `||` chain is a boolean -// slot. `a && b || c` is a left-nested chain of `binary_expression`s, so -// an operand that is itself a chain is paid by its own operator's visit. -// Read by field: a comment beside the operator is a named child too, and -// must not pay. -fn csharp_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - csharp_count_condition(&operand, conditions); - } +// tree-sitter-c-sharp wraps every argument in an `argument` node — +// `seq(optional(seq(field('name', …), ':')), optional('ref' | 'out' | +// 'in'), expression | declaration_expression)` — so the value is its +// last operand, after any named-argument label (`name: !a`) or +// passing-mode keyword. Handing the wrapper itself to the peel scored +// every argument zero, `G(!a)` included (#1537). The value is located +// by role rather than by index because both prefixes are optional +// (grammar-dispatch §3). +fn csharp_count_argument(argument: &Node, conditions: &mut f64) { + if let Some(value) = last_operand(argument) { + csharp_count_negated(&value, conditions); } } @@ -632,21 +631,23 @@ fn csharp_walk_for_conditions<'a>( use Csharp::*; let conds = &mut stats.conditions; match node.kind_id().into() { + // Fitzpatrick Rule 9: each operand of an `&&` / `||` chain is a + // boolean slot. `a && b || c` is a left-nested chain of + // `binary_expression`s, so an operand that is itself a chain is + // paid by its own operator's visit. AMPAMP | PIPEPIPE => { if let Some(chain) = ancestors .parent(node) .filter(|p| matches!(p.kind_id().into(), BinaryExpression | BinaryExpression2)) { - csharp_count_chain_operands(&chain, conds); + count_field_operands(&chain, csharp_count_condition, conds); } } - // `compute` returns as soon as `csharp_count_token_branch` fires, - // so since #1406 an `argument_list` under a `base_list` no longer - // reaches this arm. Measured harmless: the arm is dead for *every* - // argument list, because an `argument_list`'s children are - // `argument` wrappers that `csharp_wrapper_operand` declines — - // `Helper(!b)` and `Helper((b))` both score zero conditions today. - // Repairing it means revisiting that exclusion, not just this arm. + // Every `argument_list` but the one a `base_list` holds directly: + // that one is itself the branch, so `compute` scores its + // arguments on the branch path instead. A `bracketed_argument_list` + // (an indexer, `o[!a]`) is a distinct kind and is not walked, as + // Java's array index is not. ArgumentList => csharp_count_arguments(node, conds), // tree-sitter-c-sharp spells the parens of `if` / `while` / // `do … while` as anonymous tokens, NOT a wrapping @@ -754,6 +755,14 @@ impl Abc for CsharpCode { return; } if csharp_count_token_branch(node, ancestors, stats) { + // `class S(bool a) : Base(!a)`: the `argument_list` under the + // `base_list` is the branch node itself, so the walk below + // never sees it. The record spelling nests its list inside a + // `primary_constructor_base_type` and reaches the walk, so + // without this the two spellings of one base call disagree. + if node.kind_id() == Csharp::ArgumentList { + csharp_count_arguments(node, &mut stats.conditions); + } return; } if csharp_count_token_condition(node, ancestors, stats) { diff --git a/src/metrics/abc/elixir.rs b/src/metrics/abc/elixir.rs index 98db89f2..300eefa4 100644 --- a/src/metrics/abc/elixir.rs +++ b/src/metrics/abc/elixir.rs @@ -9,17 +9,16 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, last_operand, wrapped_operand}; -use crate::lang_helpers::elixir::elixir_call_keyword; +use super::{Abc, Stats, count_boolean_slot, count_field_operands, last_operand, wrapped_operand}; +use crate::lang_helpers::elixir::{elixir_applying_operator, elixir_call_keyword}; +use crate::metrics::elixir_decisions::{ + elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_sole_unguarded_pattern, + elixir_when_alternative, elixir_when_is_guard, +}; use crate::*; -/// The grammar rule an applied binary operator hangs off, as opposed to -/// `operator_identifier`, which is how one is *named* -/// (`& elixir_count_condition(&pattern, conditions), None => *conditions += 1., } @@ -191,20 +190,6 @@ fn elixir_has_else(call: &Node, code: &[u8]) -> bool { }) } -// Each operand of an `&&` / `||` / `and` / `or` chain is a boolean slot -// (Fitzpatrick Rule 9). tree-sitter-elixir parses `a && b || c` as a -// left-nested chain of `binary_operator`s, so an operand that is itself -// a chain is paid by its own operator's visit. Read by field: a comment -// beside the operator is a named child of the `binary_operator` too, -// and must not pay. -fn elixir_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - elixir_count_condition(&operand, conditions); - } - } -} - // What an Elixir `Call` contributes. The classification is by keyword // text rather than by kind, so it is a paragraph of policy rather than a // dispatch arm — the same split `java_count_token_branch` and @@ -273,8 +258,8 @@ impl Abc for ElixirCode { // (`==`, `===`, `!=`, `!==`, `<`, `>`, `<=`, `>=`, `in`, `not in`), // the boolean slots scored by `elixir_count_condition` (`if` / // `unless` predicates, guards, chain operands), one per clause of a - // clause construct (`elixir_count_clause`), and the `else` of an - // `if` / `unless` (Rule 5). + // clause construct (`elixir_count_clause`) and per `<-` clause of a + // `with`, and the `else` of an `if` / `unless` (Rule 5). // `for` / `while` are looping forms — not condition-shaped per // the issue body's literal list — so we omit them. // @@ -380,7 +365,7 @@ impl Abc for ElixirCode { // is not re-plumbed out of the predicate because that // predicate's whole job (§7) is to be one boolean the `Abc` // and `Cyclomatic` impls share. - E::When if npa::elixir_when_is_guard(node, code, ancestors) => { + E::When if elixir_when_is_guard(node, code, ancestors) => { if let Some(operator) = ancestors.parent(node) { elixir_count_guard(&operator, &mut stats.conditions); } @@ -449,9 +434,7 @@ impl Abc for ElixirCode { | E::GT | E::In | E::Notin - if ancestors - .parent(node) - .is_some_and(|parent| parent.kind() == BINARY_OPERATOR) => + if elixir_applying_operator(node, ancestors).is_some() => { stats.conditions += 1.; } @@ -460,18 +443,26 @@ impl Abc for ElixirCode { // The short-circuit operators are not counted directly // (cross-language policy, #395). Under error recovery the // token's parent can be an `ERROR` node, which has no - // operands to score. + // operands to score. `a && b || c` is a left-nested chain of + // `binary_operator`s, so an operand that is itself a chain is + // paid by its own operator's visit. E::AMPAMP | E::PIPEPIPE | E::And | E::Or => { - if let Some(chain) = ancestors - .parent(node) - .filter(|parent| parent.kind() == BINARY_OPERATOR) - { - elixir_count_chain_operands(&chain, &mut stats.conditions); + if let Some(chain) = elixir_applying_operator(node, ancestors) { + count_field_operands(&chain, elixir_count_condition, &mut stats.conditions); } } - E::StabClause if npa::elixir_clause_is_decision(node, code, ancestors) => { + E::StabClause if elixir_clause_is_decision(node, code, ancestors) => { elixir_count_clause(node, &mut stats.conditions); } + // A `with`'s `pattern <- expr` clause is a pattern-match + // condition (Rule 9), the decision `Cyclomatic` counts + // through the same gate (#1535). One flat condition: a + // pattern is never an operator that scores itself, and a + // guard on it (`{:ok, a} when g <- x`) pays its own slot + // through the `when` arm, as a guarded clause's does. + E::LTDASH if elixir_arrow_is_with_clause(node, code, ancestors) => { + stats.conditions += 1.; + } _ => {} } } diff --git a/src/metrics/abc/groovy.rs b/src/metrics/abc/groovy.rs index c67e1170..9335365a 100644 --- a/src/metrics/abc/groovy.rs +++ b/src/metrics/abc/groovy.rs @@ -10,7 +10,8 @@ )] use super::{ - Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, peel, wrapped_operand, + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, peel, wrapped_operand, }; use crate::*; @@ -366,23 +367,16 @@ fn groovy_walk_for_conditions<'a>( use Groovy::*; let conds = &mut stats.conditions; match node.kind_id().into() { - // Each operand of an `&&` / `||` chain is a boolean slot, read by - // field so a comment beside the operator does not pay. + // Each operand of an `&&` / `||` chain is a boolean slot. AMPAMP | PIPEPIPE => { if let Some(chain) = ancestors .parent(node) .filter(|p| p.kind_id() == BinaryExpression) { - for field in ["left", "right"] { - groovy_count_slot(chain.child_by_field_name(field), conds); - } - } - } - ArgumentList => { - for argument in node.children().filter(is_operand) { - groovy_count_negated(&argument, conds); + count_field_operands(&chain, groovy_count_condition, conds); } } + ArgumentList => count_each_operand(node, groovy_count_negated, conds), VariableDeclarator => groovy_count_negated_slot(node.child_by_field_name("value"), conds), AssignmentExpression => { groovy_count_negated_slot(node.child_by_field_name("right"), conds); diff --git a/src/metrics/abc/irules.rs b/src/metrics/abc/irules.rs index 163b77e1..58b8d6e8 100644 --- a/src/metrics/abc/irules.rs +++ b/src/metrics/abc/irules.rs @@ -9,7 +9,9 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, wrapped_operand}; +use super::{ + Abc, Stats, count_boolean_slot, count_each_operand, count_negated_operand, wrapped_operand, +}; use crate::*; /// The three operand slots of a `ternary_expr`, located relative to the @@ -244,8 +246,6 @@ fn irules_count_condition(condition: &Node, conditions: &mut f64) { // operand field, and inlines the parens of `($a)`. fn irules_count_chain_operands(chain: &Node, conditions: &mut f64) { if chain.kind_id() == Irules::BinopExpr { - for operand in chain.children().filter(is_operand) { - irules_count_condition(&operand, conditions); - } + count_each_operand(chain, irules_count_condition, conditions); } } diff --git a/src/metrics/abc/java.rs b/src/metrics/abc/java.rs index b01444c1..c154116c 100644 --- a/src/metrics/abc/java.rs +++ b/src/metrics/abc/java.rs @@ -10,7 +10,8 @@ )] use super::{ - Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, peel, wrapped_operand, + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, peel, wrapped_operand, }; use crate::*; @@ -96,19 +97,6 @@ fn java_count_negated(operand: &Node, conditions: &mut f64) { ); } -// Fitzpatrick Rule 9 (#403): each operand of an `&&` / `||` chain is a -// boolean slot. `a && b || c` is a left-nested chain of -// `binary_expression`s, so an operand that is itself a chain is paid by -// its own operator's visit. Read by field: a comment beside the operator -// is a named child too, and must not pay. -fn java_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - java_count_condition(&operand, conditions); - } - } -} - fn java_count_slot(slot: Option, conditions: &mut f64) { if let Some(condition) = slot { java_count_condition(&condition, conditions); @@ -311,21 +299,20 @@ fn java_walk_for_conditions<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>, s use Java::*; let conds = &mut stats.conditions; match node.kind_id().into() { - // Each operand of an `&&` / `||` chain is a boolean slot. + // Fitzpatrick Rule 9 (#403): each operand of an `&&` / `||` chain + // is a boolean slot. `a && b || c` is a left-nested chain of + // `binary_expression`s, so an operand that is itself a chain is + // paid by its own operator's visit. AMPAMP | PIPEPIPE => { if let Some(chain) = ancestors .parent(node) .filter(|p| p.kind_id() == BinaryExpression) { - java_count_chain_operands(&chain, conds); + count_field_operands(&chain, java_count_condition, conds); } } // Negated operands among method arguments. - ArgumentList => { - for argument in node.children().filter(is_operand) { - java_count_negated(&argument, conds); - } - } + ArgumentList => count_each_operand(node, java_count_negated, conds), // `if (cond)`, `while (cond)`, `do … while (cond);`, by grammar // field: a fixed index lands on a comment before the slot // (`if /*c*/ (b)`) and scores the condition zero (#1455). diff --git a/src/metrics/abc/js_family.rs b/src/metrics/abc/js_family.rs index dea1f561..ec0aba7b 100644 --- a/src/metrics/abc/js_family.rs +++ b/src/metrics/abc/js_family.rs @@ -10,8 +10,8 @@ )] use super::{ - Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, last_operand, - wrapped_operand, + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, last_operand, wrapped_operand, }; use crate::*; @@ -104,19 +104,6 @@ fn js_family_count_negated(operand: &Node, conditions: &mut f64) { ); } -// Fitzpatrick Rule 9 (#403): each operand of an `&&` / `||` chain is a -// boolean slot. `a && b || c` is a left-nested chain of -// `binary_expression`s, so an operand that is itself a chain is paid by -// its own operator's visit. Read by field: a comment beside the operator -// is a named child too, and must not pay. -fn js_family_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - js_family_count_condition(&operand, conditions); - } - } -} - // Phase-2B (issues #403 / #1102): a ternary's condition is a boolean // slot, and each branch operand a negated operand, exactly as // `java_walk_ternary` counts them. Without this the JS family scored @@ -160,13 +147,6 @@ fn js_family_walk_for(node: &Node, conditions: &mut f64) { } } -// Each argument of a call is a negated operand (`f(!a, !b)`). -fn js_family_count_arguments(arguments: &Node, conditions: &mut f64) { - for argument in arguments.children().filter(is_operand) { - js_family_count_negated(&argument, conditions); - } -} - // Generates the per-language predicate deciding whether an `=` token // initialises a `const` binding, whose initializer is part of the // declaration and therefore not an ABC assignment (Fitzpatrick). @@ -367,10 +347,17 @@ macro_rules! ts_abc_compute { stats.conditions += 1.; } // Fitzpatrick Rule 9: each operand of a `&&` / `||` - // chain is one condition (issue #403). + // chain is one condition (issue #403). `a && b || c` is a + // left-nested chain of `binary_expression`s, so an operand + // that is itself a chain is paid by its own operator's + // visit. AMPAMP | PIPEPIPE => { if let Some(chain) = ancestors.parent(node) { - js_family_count_chain_operands(&chain, &mut stats.conditions); + count_field_operands( + &chain, + js_family_count_condition, + &mut stats.conditions, + ); } } // Phase-2B (issue #403): condition slots. JS / TS @@ -395,7 +382,7 @@ macro_rules! ts_abc_compute { } // Method-argument walker for `f(!a, !b)`. Arguments => { - js_family_count_arguments(node, &mut stats.conditions); + count_each_operand(node, js_family_count_negated, &mut stats.conditions); } // `a ? !b : !c` — the ternary's own `?` token is // already counted by the condition arm above; this @@ -488,10 +475,17 @@ macro_rules! js_abc_compute { stats.conditions += 1.; } // Fitzpatrick Rule 9: each operand of a `&&` / `||` - // chain is one condition (issue #403). + // chain is one condition (issue #403). `a && b || c` is a + // left-nested chain of `binary_expression`s, so an operand + // that is itself a chain is paid by its own operator's + // visit. AMPAMP | PIPEPIPE => { if let Some(chain) = ancestors.parent(node) { - js_family_count_chain_operands(&chain, &mut stats.conditions); + count_field_operands( + &chain, + js_family_count_condition, + &mut stats.conditions, + ); } } // Phase-2B (issue #403): condition slots. Same shape @@ -508,7 +502,7 @@ macro_rules! js_abc_compute { } } Arguments => { - js_family_count_arguments(node, &mut stats.conditions); + count_each_operand(node, js_family_count_negated, &mut stats.conditions); } // `a ? !b : !c` — the ternary's own `?` token is // already counted by the condition arm above; this diff --git a/src/metrics/abc/kotlin.rs b/src/metrics/abc/kotlin.rs index b69b159e..04154e4f 100644 --- a/src/metrics/abc/kotlin.rs +++ b/src/metrics/abc/kotlin.rs @@ -9,7 +9,7 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, wrapped_operand}; +use super::{Abc, Stats, count_boolean_slot, count_field_operands, wrapped_operand}; use crate::*; // Fitzpatrick's ABC rules adapted for Kotlin syntax. Kotlin shares the @@ -175,18 +175,6 @@ fn kotlin_count_condition(condition: &Node, conditions: &mut f64) { ); } -// Fitzpatrick Rule 9 (#557): each operand of an `&&` / `||` chain is a -// boolean slot. `a && b || c` is a left-nested chain of -// `binary_expression`s, so an operand that is itself a chain is paid by -// its own operator's visit. Read by field, so a comment beside the -// operator does not pay. -fn kotlin_count_chain_operands(chain: &Node, conditions: &mut f64) { - let operands = ["left", "right"].map(|field| chain.child_by_field_name(field)); - for operand in operands.iter().flatten() { - kotlin_count_condition(operand, conditions); - } -} - // Returns true when the `when_expression` enclosing `entry` carries a // subject — `when (x) { … }` rather than `when { … }`. The distinction // decides how the entry's condition is scored (#1421): a subject-ful @@ -484,10 +472,12 @@ impl Abc for KotlinCode { // `&&` / `||` chain is one condition (issue #557). The short- // circuit operators are not counted directly (cross-language // policy, #395); the walker fires off the operator token and - // inspects the parent `binary_expression`. + // inspects the parent `binary_expression`. `a && b || c` is a + // left-nested chain, so an operand that is itself a chain is + // paid by its own operator's visit. AMPAMP | PIPEPIPE => { if let Some(parent) = ancestors.parent(node) { - kotlin_count_chain_operands(&parent, &mut stats.conditions); + count_field_operands(&parent, kotlin_count_condition, &mut stats.conditions); } } _ => {} diff --git a/src/metrics/abc/lua.rs b/src/metrics/abc/lua.rs index 179c96a5..1324a6f4 100644 --- a/src/metrics/abc/lua.rs +++ b/src/metrics/abc/lua.rs @@ -9,7 +9,10 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, wrapped_operand}; +use super::{ + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, wrapped_operand, +}; use crate::*; // Fitzpatrick's ABC rules adapted for Lua. @@ -103,27 +106,6 @@ fn lua_count_slot(slot: Option, conditions: &mut f64) { } } -// Each operand of an `and` / `or` chain is a boolean slot (Fitzpatrick -// Rule 9, #403). `a and b or c` is a left-nested chain of -// `binary_expression`s, so an operand that is itself a chain is paid by -// its own operator's visit. Read by field: a comment beside the operator -// is a named child too, and must not pay. -fn lua_count_chain_operands(chain: &Node, conditions: &mut f64) { - if chain.kind_id() == Lua::BinaryExpression { - for field in ["left", "right"] { - lua_count_slot(chain.child_by_field_name(field), conditions); - } - } -} - -// Each operand of an `expression_list` or `arguments` list is a -// negated operand. -fn lua_count_negated_operands(list: &Node, conditions: &mut f64) { - for operand in list.children().filter(is_operand) { - lua_count_negated(&operand, conditions); - } -} - impl Abc for LuaCode { fn compute<'a>( node: &Node<'a>, @@ -158,10 +140,16 @@ impl Abc for LuaCode { stats.conditions += 1.; } // Fitzpatrick Rule 9 walker: each operand of an `and` / - // `or` chain is one condition (issue #403). + // `or` chain is one condition (issue #403). `a and b or c` + // is a left-nested chain of `binary_expression`s, so an + // operand that is itself a chain is paid by its own + // operator's visit. Lua::And | Lua::Or => { - if let Some(chain) = ancestors.parent(node) { - lua_count_chain_operands(&chain, &mut stats.conditions); + if let Some(chain) = ancestors + .parent(node) + .filter(|p| p.kind_id() == Lua::BinaryExpression) + { + count_field_operands(&chain, lua_count_condition, &mut stats.conditions); } } // An `elseif` is Java's `else if`: the `else` (+1, Rule 5) and @@ -192,12 +180,12 @@ impl Abc for LuaCode { // reports zero. Bare `return` (no values) has no operand. Lua::ReturnStatement => { if let Some(expr_list) = wrapped_operand(node) { - lua_count_negated_operands(&expr_list, &mut stats.conditions); + count_each_operand(&expr_list, lua_count_negated, &mut stats.conditions); } } // `f(not a, not b)` — argument-list walker. Lua::Arguments => { - lua_count_negated_operands(node, &mut stats.conditions); + count_each_operand(node, lua_count_negated, &mut stats.conditions); } _ => {} } diff --git a/src/metrics/abc/mozcpp.rs b/src/metrics/abc/mozcpp.rs index d5f32b2f..6aba4411 100644 --- a/src/metrics/abc/mozcpp.rs +++ b/src/metrics/abc/mozcpp.rs @@ -124,7 +124,9 @@ impl Abc for MozcppCode { IfStatement | WhileStatement | DoStatement => { cpp_count_condition_slot(node, &mut stats.conditions); } - ReturnStatement => { + // `co_return` reads its value as `return` does; the + // derivation is on the Cpp twin (#1547). + ReturnStatement | CoReturnStatement => { cpp_count_return(node, &mut stats.conditions); } // `f(!a, !b)` — argument list walker. Two aliases — diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index d082355a..0589a4c4 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -10,9 +10,12 @@ )] use super::{ - Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, last_operand, + Abc, Stats, count_boolean_slot, count_each_operand, count_negated_operand, is_operand, peel, wrapped_operand, }; +use crate::lang_helpers::perl::{ + perl_and_is_operator, perl_dash_signs_keyword_key, perl_is_dash_key_word, perl_not_is_key, +}; use crate::*; // Fitzpatrick's ABC rules adapted for Perl. @@ -32,18 +35,53 @@ use crate::*; // already contributed the branch. // - Conditions: numeric and string comparison operators (`==`, `!=`, // `<`, `>`, `<=`, `>=`, `<=>`, `eq`, `ne`, `lt`, `gt`, `le`, `ge`, -// `cmp`, `=~`, `!~`), a bare `/re/` / `m{re}` match against `$_`, +// `cmp`, and `=~` / `!~` binding a match rather than an `s///` / +// `tr///` edit), a bare `/re/` / `m{re}` match against `$_`, // the ternary operator (`TernaryExpression`), // and each `elsif` / `else` clause of an `if` / `unless` // statement. Each condition slot pays one condition unless an arm // inside it already did (see `perl_count_condition`). // // The short-circuit and low-precedence logical operators (`&&`, -// `||`, `//`, `and`, `or`, `xor`) are deliberately NOT counted. +// `||`, `//`, `and`, `or`) are not counted as tokens; each operand +// of their chain is a boolean slot instead (Rule 9, issue #403). // See the module-level `Stats` doc-comment for the cross- // language policy (Fitzpatrick rules mapped from Figure 2 for C, // the closest analogue since the paper does not define rules for -// Perl; issue #395, walker tracked in #403). +// Perl; issue #395). `xor` is no chain: it evaluates both operands, +// so like Java's `^` it is a value, and the slot holding it pays +// one condition (#1536). + +// Whether `child` can occupy a Perl operand slot. tree-sitter-perl +// names its separators — `normal_comma` (`,`) and `fat_comma` (`=>`) — +// where every other grammar leaves them anonymous, so the shared +// `is_operand` took a trailing one for the list's last element: +// `if ($a > 1,)` and `if ($a > 1 =>)` scored the comma as an unproven +// slot occupant on top of the `>`, where `perl -MO=Deparse` reduces both +// to `if ($a > 1)`. +fn perl_is_operand(child: &Node) -> bool { + is_operand(child) && !matches!(child.kind_id().into(), Perl::NormalComma | Perl::FatComma) +} + +// The last operand of `node`, separators aside: the value a Perl list +// evaluates to in scalar context. +fn perl_last_operand<'a>(node: &Node<'a>) -> Option> { + node.children().filter(perl_is_operand).last() +} + +// `node` with every `( … )` around it taken off. A parenthesised operand +// is the value inside it, so `$x =~ (s/a/b/)` binds the substitution +// as `$x =~ s/a/b/` does. +fn perl_unparenthesized(node: Node<'_>) -> Node<'_> { + let mut node = node; + while node.kind_id() == Perl::Array as u16 + && let Some(inner) = perl_last_operand(&node) + { + node = inner; + } + node +} + // One step of the value peel (see `PeelStep`). // // `Array` is tree-sitter-perl's name for the `(...)` shape used BOTH as @@ -66,23 +104,43 @@ use crate::*; // match the hidden `_unary_not` supertype (`P::UnaryNot`), which the // parser never emits (grammar-dispatch item 2). // +// FIXME(#1541 upstream): an auto-quoted `not` key (`f(not => !$a)`) +// parses as that same `unary_expression`, with the `=>` in an `ERROR` +// beside the token. It is the list `("not", !$a)`, not a negation, so +// it peels like one: to its last element, the value, proving nothing. +// FIXME(#1539 upstream): an auto-quoted `and` key (`f(and => !$a)`) is +// the same shape led by `and`, and peels the same way; a real `and` +// has a left operand, so `perl_and_is_operator` tells the two apart. +// FIXME(#1545 upstream): a `-and` / `-not` key nests that shape as the +// operand of a `-` (`f(-not => !$a)`); the sign is part of the key, so +// the peel steps through it to the key. +// // No operand is read by index: a comment may sit before or after it // (`(# c` / `! # c`, #1455). fn perl_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { use Perl as P; match node.kind_id().into() { - P::Array | P::Arguments => last_operand(node).map(|o| (o, false)), + P::Array | P::Arguments => perl_last_operand(node).map(|o| (o, false)), P::ParenthesizedArgument => wrapped_operand(node).map(|o| (o, false)), P::BinaryExpression if node.is_child(P::EQ as u16) => { - last_operand(node).map(|o| (o, false)) + perl_last_operand(node).map(|o| (o, false)) } - P::UnaryExpression - if node - .child_by_field_name("operator") - .is_some_and(|op| matches!(op.kind_id().into(), P::BANG | P::Not)) => - { - wrapped_operand(node).map(|o| (o, true)) + P::UnaryExpression => { + node.child_by_field_name("operator") + .and_then(|op| match op.kind_id().into() { + P::Not if perl_not_is_key(&op, node) => { + perl_last_operand(node).map(|o| (o, false)) + } + P::And if !perl_and_is_operator(&op, node) => { + perl_last_operand(node).map(|o| (o, false)) + } + P::DASH if perl_dash_signs_keyword_key(&op, node) => { + wrapped_operand(node).map(|o| (o, false)) + } + P::BANG | P::Not => wrapped_operand(node).map(|o| (o, true)), + _ => None, + }) } _ => None, } @@ -102,37 +160,61 @@ fn perl_condition_scores_itself(expr: &Node) -> bool { match expr.kind_id().into() { P::TernaryExpression | P::PatternMatcher | P::PatternMatcherM => true, - P::BinaryExpression | P::UnaryExpression => expr.children().any(|token| { - matches!( - token.kind_id().into(), - P::EQEQ - | P::BANGEQ - | P::LT - | P::GT - | P::LTEQ - | P::GTEQ - | P::LTEQGT - | P::Eq - | P::Ne - | P::Lt - | P::Gt - | P::Le - | P::Ge - | P::Cmp - | P::EQTILDE - | P::BANGTILDE - | P::AMPAMP - | P::PIPEPIPE - | P::SLASHSLASH - | P::And - | P::Or - | P::Xor - ) - }), + P::BinaryExpression | P::UnaryExpression => { + perl_is_logical_chain(expr) + || expr.children().any(|token| match token.kind_id().into() { + P::EQTILDE | P::BANGTILDE => !perl_binding_rewrites(expr), + kind => matches!( + kind, + P::EQEQ + | P::BANGEQ + | P::LT + | P::GT + | P::LTEQ + | P::GTEQ + | P::LTEQGT + | P::Eq + | P::Ne + | P::Lt + | P::Gt + | P::Le + | P::Ge + | P::Cmp + ), + }) + } _ => false, } } +// Whether a `=~` / `!~` binding applies a substitution or transliteration +// (`$x =~ s/a/b/`, `$x =~ tr/a/b/`, `$x =~ y/a/b/`) rather than a match. +// Those edit `$x` and yield a count, so by use they are no test, exactly +// as the bare `s///` / `tr///` on `$_` are not (#1540) — while a match +// (`/re/`, `m{re}`, `qr//`, or a variable holding a pattern) is. A slot +// still pays for a rewrite binding through `perl_count_condition`, +// because this predicate also keeps `perl_condition_scores_itself` from +// claiming it. The grammar names no `left` / `right` field on a +// `binary_expression`, so the right operand is read as its last, with +// any parentheses taken off: `$x =~ (s/a/b/)` is the same edit. +// +// `$x !~ s///r` and `$x !~ tr///r` parse the same way but are Perl +// compile errors ("Using !~ with s///r doesn't make sense"); valid input +// cannot tell how they score, so no test pins them (grammar-dispatch §6). +// +// Only a `binary_expression` binds: under error recovery the token's +// parent can be an `ERROR` holding the whole file, whose last operand is +// no right operand, and reading it per token was quadratic. +fn perl_binding_rewrites(binding: &Node) -> bool { + binding.kind_id() == Perl::BinaryExpression as u16 + && perl_last_operand(binding).is_some_and(|rhs| { + matches!( + perl_unparenthesized(rhs).kind_id().into(), + Perl::SubstitutionPatternS | Perl::TransliterationTrOrY + ) + }) +} + // Scores one boolean slot — an `if` / `elsif` / `unless` / `while` / // `until` condition, a statement modifier's condition, a ternary or // C-style `for` condition, an operand of a logical chain (see @@ -168,6 +250,110 @@ fn perl_count_slot(slot: Option, conditions: &mut f64) { } } +// FIXME(#1273 upstream): tree-sitter-perl 1.1.2 lets a chaining +// comparison (`< > <= >= lt gt le ge == != eq ne`) swallow everything +// of lower precedence to its right, so `$a > $b && $c` parses as +// `$a > ($b && $c)` and `$a > $b ? 1 : 2` as `$a > ($b ? 1 : 2)`. The +// operand those trees hand to a boolean slot — `$b` — is really the +// comparison's right operand, and the real slot occupant is the +// comparison, which scores itself. Remove these helpers, and the two +// call sites' `absorbed` flags, once the pin carries the upstream +// `prec` fix. +// +// Whether the leading boolean slot of `node` — a ternary's condition or +// a logical chain's first operand — belongs to such a comparison. The +// misparse nests the stolen operand down a spine of exactly these two +// slots (`$a > $b && $c ? …` is `$a > (($b && $c) ? …)`), so the climb +// follows that spine to its root and asks whether the root directly +// follows a chaining operator. An explicit `$a > ($b ? 1 : 2)` puts an +// `array` between the two and is left alone. The token check alone +// suffices: those operators occur only in `binary_expression`, apart +// from the readline `` brackets, whose interior is never an +// expression. +// +// Two guards keep the walk linear, and neither moves a count. A slot +// that would pay nothing anyway (its occupant scores itself — on a +// left-nested `$a || $b || …` spine, every slot but the bottom one) is +// answered at once, so each spine is climbed once, from its bottom; and +// only a `binary_expression`, or an `ERROR` recovered from one, can hold +// a chaining operator, so a block, list or file the climb stops at is +// not scanned child by child. Without them every spine node climbed to +// the top and every chain scanned its statement list, both quadratic. +// The `ERROR` is the one wide parent left, holding every chain of an +// unclosed list, so the operator before `child` is found by index rather +// than by a scan from the `ERROR`'s start. +fn perl_misparse_absorbs_leading_operand<'a>( + node: &Node<'a>, + mut ancestors: impl Iterator>, +) -> bool { + use Perl as P; + + if perl_leading_slot(node) + .is_none_or(|lead| perl_condition_scores_itself(&peel(&lead, perl_wrapper_operand).0)) + { + return false; + } + let mut child = *node; + let spine_parent = ancestors.find(|parent| { + let on_spine = perl_leading_slot(parent).is_some_and(|lead| lead.id() == child.id()); + if on_spine { + child = *parent; + } + !on_spine + }); + spine_parent.is_some_and(|parent| { + (parent.kind_id() == P::BinaryExpression as u16 || parent.is_error()) + && parent + .children_before(&child) + .find(|sibling| !sibling.as_tree_sitter().is_extra()) + .is_some_and(|operator| { + matches!( + operator.kind_id().into(), + P::LT + | P::GT + | P::LTEQ + | P::GTEQ + | P::Lt + | P::Gt + | P::Le + | P::Ge + | P::EQEQ + | P::BANGEQ + | P::Eq + | P::Ne + ) + }) + }) +} + +// The leading boolean slot of a spine node — a ternary's condition or a +// logical chain's first operand — or `None` for any other node. +fn perl_leading_slot<'a>(node: &Node<'a>) -> Option> { + if node.kind_id() == Perl::TernaryExpression as u16 { + node.child_by_field_name("condition") + } else if perl_is_logical_chain(node) { + wrapped_operand(node) + } else { + None + } +} + +// A `&&` / `||` / `//` / `and` / `or` chain. tree-sitter-perl spells +// `and` as a two-operand `unary_expression`. `xor` is left out: it does +// not short-circuit, so it is a value, as `^` is (#1536). +fn perl_is_logical_chain(node: &Node) -> bool { + use Perl as P; + + matches!( + node.kind_id().into(), + P::BinaryExpression | P::UnaryExpression + ) && node.children().any(|token| match token.kind_id().into() { + P::And => perl_and_is_operator(&token, node), + P::AMPAMP | P::PIPEPIPE | P::SLASHSLASH | P::Or => true, + _ => false, + }) +} + // Phase-2B (issues #403 / #1102): a ternary's condition is a boolean // slot and each branch a negated operand, exactly as `java_walk_ternary` // counts them. Without this Perl scored `$a ? !$b : !$c` as 1 (the @@ -177,8 +363,12 @@ fn perl_count_slot(slot: Option, conditions: &mut f64) { // tree-sitter-perl names the branches `true` / `false` rather than the // C-family `consequence` / `alternative`, and all three slots are // mandatory — Perl has no short-ternary elision. -fn perl_walk_ternary(node: &Node, conditions: &mut f64) { - perl_count_slot(node.child_by_field_name("condition"), conditions); +// +// `condition_absorbed`: see `perl_misparse_absorbs_leading_operand`. +fn perl_walk_ternary(node: &Node, condition_absorbed: bool, conditions: &mut f64) { + if !condition_absorbed { + perl_count_slot(node.child_by_field_name("condition"), conditions); + } for field in ["true", "false"] { if let Some(branch) = node.child_by_field_name(field) { perl_count_negated(&branch, conditions); @@ -193,12 +383,17 @@ fn perl_walk_ternary(node: &Node, conditions: &mut f64) { // `extra`: tree-sitter-perl names no operand field on a comparison or on // the two-operand `unary_expression` it parses `$a and $b` as, which the // field-less walk is what scores at all — it scored 0. -fn perl_count_chain_operands(chain: &Node, conditions: &mut f64) { - if matches!( - chain.kind_id().into(), - Perl::BinaryExpression | Perl::UnaryExpression - ) { - for operand in chain.children().filter(is_operand) { +// +// `leading_absorbed`: see `perl_misparse_absorbs_leading_operand`. The +// `chain` is the operator token's parent, which is no chain when the +// token sits in an `ERROR` or is an auto-quoted `and` key (#1539). +fn perl_count_chain_operands(chain: &Node, leading_absorbed: bool, conditions: &mut f64) { + if perl_is_logical_chain(chain) { + for operand in chain + .children() + .filter(perl_is_operand) + .skip(usize::from(leading_absorbed)) + { perl_count_condition(&operand, conditions); } } @@ -210,52 +405,87 @@ fn perl_count_chain_operands(chain: &Node, conditions: &mut f64) { // - **Bound**: the pattern is the right operand of `$x =~ /re/` or // `$x !~ /re/`, whose operator token is already a condition. Read // through the grammar's `operator` field, so `$x ~~ /re/` — a -// smartmatch, which no arm counts — still scores the match once. +// smartmatch, which no arm counts — still scores the match once. Only +// the *right* operand is bound: `/re/ =~ $x` first matches `$_` and +// then binds that result (`perl -MO=Deparse` prints `/re/ =~ /$x/`), +// so it scores the match and the `=~` as its explicit twin +// `my $m = /re/; $m =~ $x` does. // - **`split`'s delimiter**: `split /,/, $s` and `split(/,/, $s)` hand // the pattern to `split` as a separator; nothing is matched against // `$_`, so it is no condition. Only the *first* argument is a // delimiter: a pattern elsewhere in the list is an ordinary match -// whose result `split` receives. The callee is identified by its -// `function_name` bytes (grammar-dispatch §10) — the package -// qualifier is ignored so `CORE::split` is covered too. +// whose result `split` receives. +// +// Either role is read on the pattern's outermost `( … )`, which is the +// value inside it: `$x =~ ((/re/))` binds and `split((/,/), $s)` +// delimits as their bare twins do. A call's own argument list is an +// `Array` too, but it is the call's, not a wrapper around the pattern, +// so the climb stops below one. Whether the pattern is a list's last +// operand is read forward from the pattern, which stops at the next +// operand: read off the list's last operand instead, every pattern of a +// wide `(/a/, /b/, …)` scanned the whole list. fn perl_pattern_is_bound_or_delimiter(pattern: &Node, code: &[u8], ancestors: Ancestors) -> bool { use Perl as P; - let mut climb = ancestors.iter(pattern).map(|(ancestor, _)| ancestor); - climb - .next() - .is_some_and(|parent| match parent.kind_id().into() { - P::BinaryExpression => parent + let mut up = ancestors.iter(pattern).map(|(ancestor, _)| ancestor); + let mut slot = *pattern; + let mut owner = up.next(); + let mut above = up.next(); + while let Some(paren) = owner + && paren.kind_id() == P::Array as u16 + && !paren + .children_after(&slot) + .any(|next| perl_is_operand(&next)) + && !above.is_some_and(perl_is_call_argument_parent) + { + slot = paren; + owner = above; + above = up.next(); + } + owner.is_some_and(|owner| match owner.kind_id().into() { + P::BinaryExpression => { + owner .child_by_field_name("operator") - .is_some_and(|op| matches!(op.kind_id().into(), P::EQTILDE | P::BANGTILDE)), - P::Arguments | P::Array => { - // The first operand, not the first named child: a - // comment is named, so `split( # sep⏎ /,/, $s)` read it - // as the first argument (#1455). - parent - .children() - .find(is_operand) - .is_some_and(|first| first.id() == pattern.id()) - && climb - .next() - .is_some_and(|call| perl_call_is_split(&call, code)) - } - _ => false, - }) + .is_some_and(|op| matches!(op.kind_id().into(), P::EQTILDE | P::BANGTILDE)) + && perl_last_operand(&owner).is_some_and(|rhs| rhs.id() == slot.id()) + } + // The first operand, not the first named child: a comment is + // named, so `split( # sep⏎ /,/, $s)` read it as the first + // argument (#1455). + P::Arguments | P::Array => { + owner + .children() + .find(perl_is_operand) + .is_some_and(|first| first.id() == slot.id()) + && above.is_some_and(|call| perl_call_is_split(&call, code)) + } + _ => false, + }) } -// Whether `call` is a `split` call. Every named child of the two call -// wrappers that take an argument list is the callee or an `args` -// field, so a list whose grandparent is one of them is its arguments. +// Whether `call` is a call of the builtin `split`, spelled bare or as +// `CORE::split`. Every named child of the two call wrappers that take an +// argument list is the callee or an `args` field, so a list whose +// grandparent is one of them is its arguments. The callee's package is +// read as well as its name: `My::split` is a user sub, and the pattern +// it is handed is an ordinary match (grammar-dispatch §10). fn perl_call_is_split(call: &Node, code: &[u8]) -> bool { + let text = |node: &Node| code.get(node.start_byte()..node.end_byte()); matches!( call.kind_id().into(), Perl::CallExpressionWithSpacedArgs | Perl::CallExpressionWithArgsWithBrackets ) && call .children() .find(|child| child.kind_id() == Perl::CallExpressionWithBareword as u16) - .and_then(|callee| callee.child_by_field_name("function_name")) - .is_some_and(|name| code.get(name.start_byte()..name.end_byte()) == Some(b"split")) + .is_some_and(|callee| { + callee + .child_by_field_name("function_name") + .is_some_and(|name| text(&name) == Some(b"split")) + && callee + .children() + .find(|child| child.kind_id() == Perl::PackageName as u16) + .is_none_or(|package| text(&package) == Some(b"CORE")) + }) } fn perl_is_call_argument_parent(parent: Node) -> bool { @@ -290,7 +520,8 @@ impl Abc for PerlCode { // grammar can hand us, neither being reasoning a reader must // do. Adding the guarded `<` / `>` arm for #1297 took the // count from 14 to 15, the statement-modifier arm for #1464 - // to 16, and the bare-match arm for #1467 to 17; each arm is + // to 16, the bare-match arm for #1467 to 17, and the `=~` / + // `!~` rewrite gate for #1540 added one more; each arm is // independent and self-describing like every other, and there // is no semantic boundary to split this lookup on. use Perl as P; @@ -335,7 +566,10 @@ impl Abc for PerlCode { // when this node is the outermost dispatch site. When the // bareword sits inside one of the wrappers above, the // outer node has already been counted and this child - // would double the branch tally. + // would double the branch tally. Nor is the word of a + // `-bareword` key a call (FIXME(#1545 upstream)): the + // grammar splits `$h{-foo}` into a file test on a bareword + // `oo`, and the key is a string. P::CallExpressionWithBareword if !ancestors.parent(node).is_some_and(|p| { matches!( @@ -346,7 +580,7 @@ impl Abc for PerlCode { | P::CallExpressionWithVariable | P::CallExpressionRecursive ) - }) => + }) && !perl_is_dash_key_word(node, ancestors) => { stats.branches += 1.; } @@ -365,11 +599,18 @@ impl Abc for PerlCode { | P::Le | P::Ge | P::Cmp - | P::EQTILDE - | P::BANGTILDE | P::ElseClause => { stats.conditions += 1.; } + // `=~` / `!~` bind a match, a test, or a rewrite, an edit + // that scores only in a slot (see `perl_binding_rewrites`). + P::EQTILDE | P::BANGTILDE + if !ancestors + .parent(node) + .is_some_and(|binding| perl_binding_rewrites(&binding)) => + { + stats.conditions += 1.; + } // Counts `<` / `>` only as the operator token of a // `binary_expression`, the allowlist polarity the rest of // the workspace uses. Ungated, a readline scored two @@ -398,8 +639,10 @@ impl Abc for PerlCode { // written (#1467) — `my $r = /^#/` levels with // `my $r = ($x =~ /^#/)`. `perl_condition_scores_itself` // lists it, or every slot would score it a second time. - // `s///` and `tr///` stay out: they edit - // `$_` and yield a count, a policy question left to #1475. + // `s///` and `tr///` stay out: they edit `$_` and yield a + // count, so by use they are no test. A slot still pays for + // either (#1475), and the bound `$x =~ s///` agrees with the + // bare form (#1540, `perl_binding_rewrites`). P::PatternMatcher | P::PatternMatcherM if !perl_pattern_is_bound_or_delimiter(node, code, ancestors) => { @@ -408,10 +651,12 @@ impl Abc for PerlCode { // Fitzpatrick Rule 9 walker: each operand of a Perl // short-circuit / low-precedence logical chain is one // condition (issue #403). Covers `&&`, `||`, `//`, - // `and`, `or`, `xor`. - P::AMPAMP | P::PIPEPIPE | P::SLASHSLASH | P::And | P::Or | P::Xor => { - if let Some(chain) = ancestors.parent(node) { - perl_count_chain_operands(&chain, &mut stats.conditions); + // `and`, `or` — not `xor`, which is a value (#1536). + P::AMPAMP | P::PIPEPIPE | P::SLASHSLASH | P::And | P::Or => { + let mut above = ancestors.iter(node).map(|(ancestor, _)| ancestor); + if let Some(chain) = above.next() { + let absorbed = perl_misparse_absorbs_leading_operand(&chain, above); + perl_count_chain_operands(&chain, absorbed, &mut stats.conditions); } } // An `elsif` is Java's `else if`: the `else` (+1, Rule 5) and @@ -441,9 +686,7 @@ impl Abc for PerlCode { .parent(node) .is_some_and(perl_is_call_argument_parent) => { - for argument in node.children().filter(is_operand) { - perl_count_negated(&argument, &mut stats.conditions); - } + count_each_operand(node, perl_count_negated, &mut stats.conditions); } // `$a ? !$b : !$c`. Unlike the C family, this dispatcher // has no `?`-token arm — the grammar emits the token, but @@ -452,7 +695,9 @@ impl Abc for PerlCode { // and adds the three operand slots (issue #1102). P::TernaryExpression => { stats.conditions += 1.; - perl_walk_ternary(node, &mut stats.conditions); + let above = ancestors.iter(node).map(|(ancestor, _)| ancestor); + let absorbed = perl_misparse_absorbs_leading_operand(node, above); + perl_walk_ternary(node, absorbed, &mut stats.conditions); } // Phase-2B (issue #403): condition slots, each read by the // grammar's `condition` field, not at child(1): a comment diff --git a/src/metrics/abc/php.rs b/src/metrics/abc/php.rs index 1d62282c..70fcdb8c 100644 --- a/src/metrics/abc/php.rs +++ b/src/metrics/abc/php.rs @@ -9,7 +9,10 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, wrapped_operand}; +use super::{ + Abc, Stats, count_boolean_slot, count_field_operands, count_negated_operand, is_operand, + wrapped_operand, +}; use crate::*; // One step of the value peel (see `PeelStep`). A parenthesis, a cast @@ -45,7 +48,7 @@ fn php_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { // Whether an arm of `compute` already charges `expr` (already peeled) as // a condition: a ternary (its own arm), a comparison or `instanceof` -// (the token arm), or an `&&` / `||` / `and` / `or` / `xor` chain, whose +// (the token arm), or an `&&` / `||` / `and` / `or` chain, whose // operands each pay through `php_count_condition`. `??` is no condition // token in PHP, so a slot holding one pays for it. fn php_condition_scores_itself(expr: &Node) -> bool { @@ -70,7 +73,6 @@ fn php_condition_scores_itself(expr: &Node) -> bool { | PIPEPIPE | And | Or - | Xor ) }), _ => false, @@ -79,7 +81,7 @@ fn php_condition_scores_itself(expr: &Node) -> bool { // Scores one boolean slot — an `if` / `elseif` / `while` / `do` / `for` // condition, a ternary condition, an operand of an `&&` / `||` / `and` / -// `or` / `xor` chain (see `count_boolean_slot`). PHP is truthy-valued, +// `or` chain (see `count_boolean_slot`). PHP is truthy-valued, // so `if (-$x)`, `if ($x + 1)`, `if ($y = $x)`, `if (A::B)` and // `if ($a ?? $b)` are each a decision, and each scored 0 while the slot // paid only for a fixed list of terminal kinds (#1526). @@ -156,17 +158,6 @@ fn php_count_arguments(arguments: &Node, conditions: &mut f64) { } } -// Fitzpatrick Rule 9 (#403): each operand of an `&&` / `||` / `and` / -// `or` / `xor` chain is a boolean slot. `$a && $b || $c` is a -// left-nested chain of `binary_expression`s, so an operand that is -// itself a chain is paid by its own operator's visit. Read by field: a -// comment beside the operator is a named child too, and must not pay. -fn php_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - php_count_slot(chain.child_by_field_name(field), conditions); - } -} - impl Abc for PhpCode { fn compute<'a>( node: &Node<'a>, @@ -230,18 +221,22 @@ impl Abc for PhpCode { stats.conditions += 1.; } // Fitzpatrick Rule 9: each operand of a `&&` / `||` / `and` - // / `or` / `xor` chain is one condition (issue #403). PHP - // exposes both the punctuation forms (`&&`, `||`) and the - // low-precedence keyword forms (`and`, `or`, `xor`) as + // / `or` chain is one condition (issue #403). `xor` is no + // chain: it evaluates both operands, so like `^` it is a + // value and its slot pays once (#1536). PHP exposes both + // the punctuation forms (`&&`, `||`) and the + // low-precedence keyword forms (`and`, `or`) as // distinct tokens inside `binary_expression`; both fire // the walker so `connect() or die();`-style idiom counts - // the same as `connect() || die();`. - AMPAMP | PIPEPIPE | And | Or | Xor => { + // the same as `connect() || die();`. `$a && $b || $c` is a + // left-nested chain of `binary_expression`s, so an operand + // that is itself a chain is paid by its own operator's visit. + AMPAMP | PIPEPIPE | And | Or => { if let Some(chain) = ancestors .parent(node) .filter(|p| p.kind_id() == BinaryExpression) { - php_count_chain_operands(&chain, &mut stats.conditions); + count_field_operands(&chain, php_count_condition, &mut stats.conditions); } } // An `elseif` is Java's `else if` — and PHP's own two-word diff --git a/src/metrics/abc/python.rs b/src/metrics/abc/python.rs index a2da25f3..0b016979 100644 --- a/src/metrics/abc/python.rs +++ b/src/metrics/abc/python.rs @@ -9,7 +9,7 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, peel, wrapped_operand}; +use super::{Abc, Stats, count_boolean_slot, count_field_operands, peel, wrapped_operand}; use crate::*; // Fitzpatrick's ABC rules adapted for Python. @@ -90,19 +90,6 @@ fn python_count_condition(condition: &Node, conditions: &mut f64) { ); } -// Each operand of an `and` / `or` chain is a boolean slot (Fitzpatrick -// Rule 9, #403). tree-sitter-python parses `a and b or c` as a -// left-nested chain of `boolean_operator`s, so an operand that is itself -// a chain is paid by its own operator's visit. Read by field: a comment -// beside the operator is a named child too, and must not pay. -fn python_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - python_count_condition(&operand, conditions); - } - } -} - // Phase-2B (issue #1161): the condition slot of a Python conditional // expression, ` if else `. Before // this, `a if c() else b` reported 1 — the `ConditionalExpression` node @@ -258,12 +245,15 @@ impl Abc for PythonCode { // `or` chain is one condition (issue #403). The `And` / // `Or` keyword tokens live inside a `boolean_operator` // wrapper which the walker iterates as the parent list. + // `a and b or c` is a left-nested chain of + // `boolean_operator`s, so an operand that is itself a chain is + // paid by its own operator's visit. And | Or => { if let Some(chain) = ancestors .parent(node) .filter(|p| p.kind_id() == BooleanOperator) { - python_count_chain_operands(&chain, &mut stats.conditions); + count_field_operands(&chain, python_count_condition, &mut stats.conditions); } } // An `elif` is Java's `else if`: the `else` (+1, Rule 5) and diff --git a/src/metrics/abc/ruby.rs b/src/metrics/abc/ruby.rs index 54f8ec25..05915c34 100644 --- a/src/metrics/abc/ruby.rs +++ b/src/metrics/abc/ruby.rs @@ -9,7 +9,9 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_negated_operand, last_operand}; +use super::{ + Abc, Stats, count_boolean_slot, count_field_operands, count_negated_operand, last_operand, +}; use crate::*; // Ruby ABC rules follow the Fitzpatrick paper's spirit, adapted to @@ -84,22 +86,6 @@ fn ruby_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { } } -// Fitzpatrick Rule 9 walker (#557): each operand of an `&&` / `||` / -// `and` / `or` chain is a boolean slot, scored by `ruby_count_condition`. -// tree-sitter-ruby parses `a && b || c` as a left-nested chain of -// `binary` nodes (aliased `Binary`..`Binary3`, lesson #2), so an operand -// that is itself a chain is paid by its own operator's visit. -// -// The operands are read by field: a comment between operand and operator -// is a named child of the `binary` too, and must not pay. -fn ruby_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - ruby_count_condition(&operand, conditions); - } - } -} - // Scores one boolean slot — the predicate of every `if` / `unless` / // `while` / `until` form (block and modifier), a `case … in` guard, a // ternary condition, each pattern of a subject-less `when`, and each @@ -369,13 +355,16 @@ impl Abc for RubyCode { // (cross-language policy, #395); the keyword forms `and` / `or` // get the same treatment as `&&` / `||`. Under error recovery // the token's parent can be an `ERROR` node (`f(&&)`), which - // has no operands to score. + // has no operands to score. tree-sitter-ruby parses + // `a && b || c` as a left-nested chain of `binary` nodes + // (aliased `Binary`..`Binary3`, lesson #2), so an operand that + // is itself a chain is paid by its own operator's visit. AMPAMP | PIPEPIPE | And | Or => { if let Some(chain) = ancestors .parent(node) .filter(|p| matches!(p.kind_id().into(), Binary | Binary2 | Binary3)) { - ruby_count_chain_operands(&chain, &mut stats.conditions); + count_field_operands(&chain, ruby_count_condition, &mut stats.conditions); } } // `a ? !b : !c` — the ternary's own `?` token is already diff --git a/src/metrics/abc/rust.rs b/src/metrics/abc/rust.rs index 6a0eb8ff..4d60b66d 100644 --- a/src/metrics/abc/rust.rs +++ b/src/metrics/abc/rust.rs @@ -10,8 +10,8 @@ )] use super::{ - Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, last_operand, - wrapped_operand, + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, last_operand, wrapped_operand, }; use crate::*; @@ -92,13 +92,6 @@ fn rust_count_slot(slot: Option, conditions: &mut f64) { } } -// Each argument of a call is a negated operand (`m(!a, !b)`). -fn rust_count_arguments(arguments: &Node, conditions: &mut f64) { - for argument in arguments.children().filter(is_operand) { - rust_count_negated(&argument, conditions); - } -} - // The conditions a `match_arm` contributes: the arm itself, plus its // guard. // @@ -151,20 +144,14 @@ fn rust_count_match_arm(node: &Node, conditions: &mut f64) { // 3). Its `let` operands score themselves. fn rust_count_chain_operands(token: &Node, chain: &Node, conditions: &mut f64) { if chain.kind_id() == Rust::BinaryExpression { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - rust_count_condition(&operand, conditions); - } - } + count_field_operands(chain, rust_count_condition, conditions); } else if matches!(chain.kind_id().into(), Rust::LetChain | Rust::LetChain2) && chain .children() .find(|child| child.kind_id() == Rust::AMPAMP) .is_some_and(|first| first.id() == token.id()) { - for operand in chain.children().filter(is_operand) { - rust_count_condition(&operand, conditions); - } + count_each_operand(chain, rust_count_condition, conditions); } } @@ -278,7 +265,8 @@ impl Abc for RustCode { } // Method-argument walker: `m(!a, !b)` contributes one // condition per negated argument. - Arguments => rust_count_arguments(node, &mut stats.conditions), + // Each argument of a call is a negated operand (`m(!a, !b)`). + Arguments => count_each_operand(node, rust_count_negated, &mut stats.conditions), _ => {} } } diff --git a/src/metrics/abc/slot_pays_tests.rs b/src/metrics/abc/slot_pays_tests.rs index d7e38095..8e3575cf 100644 --- a/src/metrics/abc/slot_pays_tests.rs +++ b/src/metrics/abc/slot_pays_tests.rs @@ -41,6 +41,32 @@ fn assert_rows(lang: LANG, template: &str, rows: &[(&str, u64, u64)]) { } } +/// Asserts each of #1475's constructs pays its `if` slot and its `&&` +/// chain-operand slot exactly as the identifier `control` does: one +/// condition and one decision in the `if`, two of each in the chain. +/// The control row runs first, so a template that stopped measuring +/// fails there rather than in every construct row. +#[cfg(any( + feature = "csharp", + feature = "groovy", + feature = "perl", + feature = "ruby", + feature = "elixir" +))] +#[track_caller] +fn assert_level_with_control( + lang: LANG, + [if_slot, chain_slot]: [&str; 2], + control: &str, + constructs: &[&str], +) { + let predicates = || std::iter::once(control).chain(constructs.iter().copied()); + let in_if: Vec<_> = predicates().map(|p| (p, 1, 1)).collect(); + assert_rows(lang, if_slot, &in_if); + let in_chain: Vec<_> = predicates().map(|p| (p, 2, 2)).collect(); + assert_rows(lang, chain_slot, &in_chain); +} + #[test] #[cfg(feature = "python")] fn python_slots_pay_for_any_predicate() { @@ -180,6 +206,13 @@ fn assert_js_family_slots(lang: LANG) { ("!(b > 1)", 1, 0), ], ); + // A call argument is no slot either; the negation still counts, + // and `this` scores like an identifier there too (#1475). + assert_rows( + lang, + &function.replace(SLOT, "g(PRED);"), + &[("b", 0, 0), ("!b", 1, 0), ("!this", 1, 0)], + ); } #[test] @@ -259,6 +292,8 @@ const C_FAMILY_IF_ROWS: &[(&str, u64, u64)] = &[ ("!(b > 1)", 1, 1), ("b > 1", 1, 1), ("!b", 1, 1), + // A negated literal is a negation like any other (#1469). + ("!\"s\"", 1, 1), // A ternary predicate is counted by its own `?` and pays its // condition slot; the `if` slot must not pay a third time. ("a ? b : y", 2, 2), @@ -301,6 +336,7 @@ fn assert_c_family_slots(lang: LANG) { ("-b", 0, 0), ("!b", 1, 0), ("!*p", 1, 0), + ("!\"s\"", 1, 0), ("!(b > 1)", 1, 0), ], ); @@ -692,6 +728,31 @@ fn csharp_slots_pay_for_any_predicate() { ); } +/// The C# spellings #1475 measured at zero before #1526 made every slot +/// pay: `default(bool)`, the `checked` wrapper (and its `unchecked` +/// twin), an assignment whose value is a call, and an object creation +/// converted to `bool` by a user-defined operator. +#[test] +#[cfg(feature = "csharp")] +fn csharp_1475_constructs_pay_their_slot() { + let function = "class C { bool y; bool f(bool a, bool b) { PRED return a; } }"; + assert_level_with_control( + LANG::Csharp, + [ + &function.replace(SLOT, "if (PRED) { g(); }"), + &function.replace(SLOT, "if (a && (PRED)) { g(); }"), + ], + "b", + &[ + "default(bool)", + "checked(b)", + "unchecked(b)", + "y = C()", + "new Wrapper()", + ], + ); +} + /// PHP is truthy-valued, so every predicate is a decision. `??` is no /// condition token in PHP, so a slot holding one pays for it. #[test] @@ -726,7 +787,8 @@ fn php_slots_pay_for_any_predicate() { ("$a && $b", 2, 2), ("$a && -$x", 2, 2), ("$a and -$x", 2, 2), - ("$a xor -$x", 2, 2), + // `xor` is a value, as `^` is: one slot, no chain (#1536). + ("$a xor -$x", 1, 1), ("$a || ($y = $x > 1)", 2, 2), // A ternary scores itself through its own arm. ("$a ? $b : $x", 2, 2), @@ -887,6 +949,24 @@ fn groovy_slots_pay_for_any_predicate() { ); } +/// #1475's Groovy rows: an object creation, which JS and PHP already +/// counted, and a spread-dot navigation, whose list result is tested +/// for emptiness. +#[test] +#[cfg(feature = "groovy")] +fn groovy_1475_constructs_pay_their_slot() { + let function = "class C { def f(a, b) { PRED; return a } }\n"; + assert_level_with_control( + LANG::Groovy, + [ + &function.replace(SLOT, "if (PRED) { g() }"), + &function.replace(SLOT, "if (a && (PRED)) { g() }"), + ], + "b", + &["new Foo()", "a*.b"], + ); +} + /// Perl is truthy-valued, so every predicate is a decision. #[test] #[cfg(feature = "perl")] @@ -917,6 +997,10 @@ fn perl_slots_pay_for_any_predicate() { // two-operand `unary_expression`; it scored 0. ("$a and $b", 2, 2), ("$a and -$x", 2, 2), + // `xor` is a value, as `^` is: one slot, no chain (#1536). + ("$a ^ $b", 1, 1), + ("$a xor $b", 1, 1), + ("$a xor -$x", 1, 1), ], ); for slot in [ @@ -972,6 +1056,70 @@ fn perl_elsif_is_an_else_and_a_slot() { ); } +/// #1475's Perl rows: the substitution and transliteration operators +/// (all three spellings), an anonymous sub, and the array and hash +/// variables in both their named and dereferencing forms. `@$b` was +/// the case #1475 called the clearest defect, since `@a` scored and it +/// did not. Each pays a slot, the `=~` / `!~`-bound rewrites included: +/// outside one they score nothing (#1540), so the slot is all they pay. +#[test] +#[cfg(feature = "perl")] +fn perl_1475_constructs_pay_their_slot() { + let function = "sub f { my ($a, $b, $h) = @_; my @a; my %h; PRED }\n"; + assert_level_with_control( + LANG::Perl, + [ + &function.replace(SLOT, "if (PRED) { g(); }"), + &function.replace(SLOT, "if ($a && (PRED)) { g(); }"), + ], + "$b", + &[ + "s/x/y/", + "tr/a/b/", + "y/a/b/", + "$b =~ s/x/y/", + "$b !~ s/x/y/", + "$b =~ tr/a/b/", + "sub {1}", + "@a", + "@$b", + "%h", + "%$h", + ], + ); +} + +/// #1475's Ruby rows: both lambda spellings and a range. +#[test] +#[cfg(feature = "ruby")] +fn ruby_1475_constructs_pay_their_slot() { + assert_level_with_control( + LANG::Ruby, + [ + "def f(a, b)\n if PRED\n g\n end\nend\n", + "def f(a, b)\n if a && (PRED)\n g\n end\nend\n", + ], + "b", + &["lambda {}", "-> {}", "(1..2)"], + ); +} + +/// #1475's Elixir rows: an anonymous function and a function capture. +#[test] +#[cfg(feature = "elixir")] +fn elixir_1475_constructs_pay_their_slot() { + let function = "defmodule M do\n def f(a, b) do\n PRED\n end\nend\n"; + assert_level_with_control( + LANG::Elixir, + [ + &function.replace(SLOT, "if PRED do\n g()\n end"), + &function.replace(SLOT, "if a && (PRED) do\n g()\n end"), + ], + "b", + &["fn -> 1 end", "&f/1"], + ); +} + /// Tcl's `expr` tests any number for non-zero, so every predicate is a /// decision. iRules is its dialect and shares every row. #[cfg(any(feature = "tcl", feature = "irules"))] diff --git a/src/metrics/abc/tcl.rs b/src/metrics/abc/tcl.rs index cf1fcc07..43918847 100644 --- a/src/metrics/abc/tcl.rs +++ b/src/metrics/abc/tcl.rs @@ -9,7 +9,9 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, wrapped_operand}; +use super::{ + Abc, Stats, count_boolean_slot, count_each_operand, count_negated_operand, wrapped_operand, +}; use crate::*; // Names of Tcl commands that mutate a variable. Each invocation of @@ -114,9 +116,7 @@ fn tcl_count_condition(condition: &Node, conditions: &mut f64) { // operand field, and inlines the parens of `($a)`. fn tcl_count_chain_operands(chain: &Node, conditions: &mut f64) { if chain.kind_id() == Tcl::BinopExpr { - for operand in chain.children().filter(is_operand) { - tcl_count_condition(&operand, conditions); - } + count_each_operand(chain, tcl_count_condition, conditions); } } diff --git a/src/metrics/bash_test_connective_tests.rs b/src/metrics/bash_test_connective_tests.rs new file mode 100644 index 00000000..fe1e3710 --- /dev/null +++ b/src/metrics/bash_test_connective_tests.rs @@ -0,0 +1,222 @@ +//! A `[ … ]` test scores like its `&&` / `||` twins (#1536). +//! +//! Bash joins two tests three ways: `[ X -a Y ]`, `[ X ] && [ Y ]` and +//! `[[ X && Y ]]` (and `-o` / `||` likewise). Before #1536 the first +//! scored no cyclomatic or cognitive decision for its `-a`, ABC counted +//! that `-a` as a comparison, and a `=` comparison scored no ABC +//! condition in any spelling. Every row groups the three spellings, +//! and each must score the row's decisions. + +use std::cell::Cell; + +use crate::test_support::check_func_space_only; +use crate::*; + +/// `[cyclomatic, abc conditions, cognitive]` sums of the root space. +/// The cyclomatic sum carries the unit's and the function's base of 1 +/// each. Bash ABC pays one condition for the `if` and one for each +/// comparison, and nothing for a connective. +#[cfg(feature = "bash")] +type Decisions = [u64; 3]; + +#[cfg(feature = "bash")] +fn measure(body: &str) -> Decisions { + let out = Cell::new([0; 3]); + check_func_space_only::( + &format!("f() {{\n {body}\n}}\n"), + "foo.sh", + &[Metric::Cyclomatic, Metric::Abc, Metric::Cognitive], + |space| { + let m = &space.metrics; + out.set([ + m.cyclomatic.cyclomatic_sum(), + m.abc.conditions_sum(), + m.cognitive.cognitive_sum(), + ]); + }, + ); + out.get() +} + +#[cfg(feature = "bash")] +#[test] +fn bash_test_connectives_score_like_their_twins_1536() { + // `(single_bracket, chained_commands, double_bracket, decisions)`. + let rows: [(&str, &str, &str, Decisions); 8] = [ + ( + r#"if [ "$a" = 1 -a "$b" = 2 ]; then :; fi"#, + r#"if [ "$a" = 1 ] && [ "$b" = 2 ]; then :; fi"#, + r#"if [[ "$a" = 1 && "$b" = 2 ]]; then :; fi"#, + [4, 3, 2], + ), + ( + r#"if [ "$a" == 1 -a "$b" == 2 ]; then :; fi"#, + r#"if [ "$a" == 1 ] && [ "$b" == 2 ]; then :; fi"#, + r#"if [[ "$a" == 1 && "$b" == 2 ]]; then :; fi"#, + [4, 3, 2], + ), + ( + r#"if [ "$a" -eq 1 -a "$b" -eq 2 ]; then :; fi"#, + r#"if [ "$a" -eq 1 ] && [ "$b" -eq 2 ]; then :; fi"#, + r#"if [[ "$a" -eq 1 && "$b" -eq 2 ]]; then :; fi"#, + [4, 3, 2], + ), + ( + r#"if [ "$a" = 1 -o "$b" = 2 ]; then :; fi"#, + r#"if [ "$a" = 1 ] || [ "$b" = 2 ]; then :; fi"#, + r#"if [[ "$a" = 1 || "$b" = 2 ]]; then :; fi"#, + [4, 3, 2], + ), + // A switch of operation is a second cognitive sequence in every + // spelling. + ( + r#"if [ "$a" = 1 -a "$b" = 2 -o "$c" = 3 ]; then :; fi"#, + r#"if [ "$a" = 1 ] && [ "$b" = 2 ] || [ "$c" = 3 ]; then :; fi"#, + r#"if [[ "$a" = 1 && "$b" = 2 || "$c" = 3 ]]; then :; fi"#, + [5, 4, 3], + ), + // `-a` and `-o` key to the symbols they stand for: a run of one + // is a single sequence. The grammar nests `=`, `==` and `!=` + // around the connectives (`"$a" = (1 -o "$b") = …`), so these + // rows also pin `ComparisonRuns`; `[[ … = … || … ]]` scored 3 + // before #1536 for the same reason. + ( + r#"if [ "$a" = 1 -o "$b" = 2 -o "$c" = 3 ]; then :; fi"#, + r#"if [ "$a" = 1 ] || [ "$b" = 2 ] || [ "$c" = 3 ]; then :; fi"#, + r#"if [[ "$a" = 1 || "$b" = 2 || "$c" = 3 ]]; then :; fi"#, + [5, 4, 2], + ), + ( + r#"if [ "$a" == 1 -o "$b" == 2 -o "$c" == 3 ]; then :; fi"#, + r#"if [ "$a" == 1 ] || [ "$b" == 2 ] || [ "$c" == 3 ]; then :; fi"#, + r#"if [[ "$a" == 1 || "$b" == 2 || "$c" == 3 ]]; then :; fi"#, + [5, 4, 2], + ), + ( + r#"if [ "$a" != 1 -o "$b" != 2 -o "$c" != 3 ]; then :; fi"#, + r#"if [ "$a" != 1 ] || [ "$b" != 2 ] || [ "$c" != 3 ]; then :; fi"#, + r#"if [[ "$a" != 1 || "$b" != 2 || "$c" != 3 ]]; then :; fi"#, + [5, 4, 2], + ), + ]; + for (single, chained, double, decisions) in rows { + assert!( + single.contains(" -a ") || single.contains(" -o "), + "`{single}` lost its connective" + ); + for body in [single, chained, double] { + assert_eq!(measure(body), decisions, "`{body}`"); + } + } +} + +/// What the gates must leave alone: a unary `-a FILE` is a file test +/// like `-f FILE`, a `=` keeps comparing however the test nests it, `=` +/// inside `(( … ))` is an assignment, and a correctly nested switch of +/// operation is still three sequences. +#[cfg(feature = "bash")] +#[test] +fn bash_test_connective_gates_keep_their_controls_1536() { + // `(body, twin, decisions)`; `twin` spells the same test another + // way and must agree. + let rows: [(&str, &str, Decisions); 6] = [ + ( + "if [ -a x ]; then :; fi", + "if [ -f x ]; then :; fi", + [3, 2, 1], + ), + ( + r#"if [ "$a" = 1 ]; then :; fi"#, + r#"if [ "$a" == 1 ]; then :; fi"#, + [3, 2, 1], + ), + ( + r#"if [[ ( "$a" = 1 ) ]]; then :; fi"#, + r#"if [[ ( "$a" == 1 ) ]]; then :; fi"#, + [3, 2, 1], + ), + ( + r#"if [[ ! ( "$a" = 1 ) ]]; then :; fi"#, + r#"if [[ ! ( "$a" == 1 ) ]]; then :; fi"#, + [3, 2, 1], + ), + ("(( x = 1 ))", "x=1", [2, 0, 0]), + ( + r#"if [[ ( "$a" = 1 || "$b" = 2 ) && ( "$c" = 3 || "$d" = 4 ) ]]; then :; fi"#, + r#"if [[ ( "$a" == 1 || "$b" == 2 ) && ( "$c" == 3 || "$d" == 4 ) ]]; then :; fi"#, + [6, 5, 4], + ), + ]; + for (body, twin, decisions) in rows { + assert_eq!( + measure(twin), + decisions, + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure(body), + decisions, + "`{body}` must score like `{twin}`" + ); + } +} + +/// An outer comparison run resumes after a nested one: past the `&&` +/// group, `"$c" = 3` continues the `||` run `"$b" = 2` restarted, so the +/// two are one sequence. The mis-nested tree scores a mixed run in +/// textual order (`ComparisonRuns`), one more than the `==` twin's 3, so +/// the value here is the one the ancestor climb `ComparisonRuns` +/// replaced scored; a tracker that forgot the outer run at the group +/// scores 5. +#[cfg(feature = "bash")] +#[test] +fn bash_comparison_run_resumes_after_a_nested_run_1536() { + assert_eq!( + measure( + r#"if [[ "$a" = 1 || ( "$x" = 5 && "$y" = 6 ) || "$b" = 2 || "$c" = 3 || "$d" = 4 ]]; then :; fi"# + ), + [8, 7, 4] + ); +} + +/// `=~` shares `=`'s precedence, so tree-sitter-bash mis-nests a run of +/// regex matches the same way (`"$a" =~ ($re || "$b") =~ …`), and the run +/// must stay one sequence, as its `==` twin's is. The other two rows pin +/// what the run tracking and the `=` count reproduce: a run that resumes +/// after a nested one keeps its sequences, and an assignment inside a +/// test's `$(( … ))` is no comparison. +#[cfg(feature = "bash")] +#[test] +fn bash_comparison_runs_score_like_their_twins_1536() { + // `(body, twin, decisions)`; `twin` spells the same test another + // way and must agree. + let rows: [(&str, &str, Decisions); 3] = [ + ( + r#"if [[ "$a" =~ $re || "$b" =~ $re || "$c" =~ $re ]]; then :; fi"#, + r#"if [[ "$a" == $re || "$b" == $re || "$c" == $re ]]; then :; fi"#, + [5, 4, 2], + ), + ( + "if [[ ( $a = 1 || $b = 2 ) && $c = 3 || $d = 4 ]]; then :; fi", + "if [[ ( $a == 1 || $b == 2 ) && $c == 3 || $d == 4 ]]; then :; fi", + [6, 5, 4], + ), + ( + "if [[ $(( x = 1 )) -eq 1 ]]; then :; fi", + "if [[ $(( x + 1 )) -eq 1 ]]; then :; fi", + [3, 2, 1], + ), + ]; + for (body, twin, decisions) in rows { + assert_eq!( + measure(twin), + decisions, + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure(body), + decisions, + "`{body}` must score like `{twin}`" + ); + } +} diff --git a/src/metrics/cognitive.rs b/src/metrics/cognitive.rs index b1764322..c79ad483 100644 --- a/src/metrics/cognitive.rs +++ b/src/metrics/cognitive.rs @@ -55,6 +55,7 @@ pub struct Stats { total_space_functions: usize, boolean_seq: BoolSequence, constraint: ConstraintReach, + bash_runs: bash::ComparisonRuns, } impl Default for Stats { @@ -68,6 +69,7 @@ impl Default for Stats { total_space_functions: 1, boolean_seq: BoolSequence::default(), constraint: ConstraintReach::default(), + bash_runs: bash::ComparisonRuns::default(), } } } @@ -211,15 +213,32 @@ fn compute_booleans_with bool>(node: &Node, stats: &mut Stats, is_ /// changes which operator nests under which, and the parse already /// carries that, so `a || b and c` is `(a || b) and c` and still /// switches operation. Operators that are a different operation -/// (`xor`, `??`, `//`) keep keys of their own (#1530). +/// (`??`, `//`) keep keys of their own (#1530); `xor`, like `^`, is no +/// sequence operator at all (#1536). fn compute_booleans_keyed Option>( node: &Node, stats: &mut Stats, sequence_key: F, ) { - let enclosing_end = node.end_byte(); + compute_booleans_by_node(node, node.end_byte(), stats, |child| { + sequence_key(child.kind_id()) + }); +} + +/// [`compute_booleans_keyed`] for an operator whose kind alone does not +/// say which operation it is — Bash's `-a` / `-o` share `test_operator` +/// with every comparison, so the key reads the child itself — and for a +/// sequence whose extent is not `node`'s own span. `enclosing_end` is +/// where the sequence `node` belongs to ends: an operator inside it +/// continues a like operator seen earlier in it. +fn compute_booleans_by_node Option>( + node: &Node, + enclosing_end: usize, + stats: &mut Stats, + sequence_key: F, +) { for child in node.children() { - if let Some(key) = sequence_key(child.kind_id()) { + if let Some(key) = sequence_key(&child) { stats.structural = stats .boolean_seq @@ -3976,6 +3995,153 @@ mod tests { ); } + // A `let`-`else` is an `if let` that diverges instead of nesting the + // happy path, so it scores what its `if let` twin scores: +1 plus the + // current nesting, with a level for everything under it (#1548). + // + // Each `le_*` fn is paired with an `il_*` twin that spells the same + // control flow with `if let`, and the expected values were measured + // on the twins *before* the fix (the twins do not move). Before it, + // the let-else paid a flat +1 through the shared `else` token arm: + // `le_lp` 2, `le_d2` 4, `le_d3` 7, `le_two` 3, the closure 1 and + // `le_in` 2. `le_in` pins that the `else` block is nested like the + // twin's body (its inner `if` pays 2), `ie` that an `if … else` still + // pays its `else`, and the closure rows that the lambda level reaches + // the let-else. `value` is the fn's own score and `sum` folds in the + // closure space it encloses. + #[cfg(feature = "rust")] + #[test] + fn rust_let_else_scores_like_its_if_let_twin_1548() { + let src = "fn le_lp(xs: &[Option]) { + for y in xs { + let Some(a) = y else { continue }; + k(a); + } +} +fn il_lp(xs: &[Option]) { + for y in xs { + if let Some(a) = y { k(a); } + } +} +fn le_top(x: i32) -> i32 { + let Ok(a) = g(x) else { return 0 }; + a +} +fn il_top(x: i32) -> i32 { + if let Ok(a) = g(x) { return a; } + 0 +} +fn le_d2(xs: &[Vec>]) { + for y in xs { + for z in y { + let Some(a) = z else { continue }; + k(a); + } + } +} +fn il_d2(xs: &[Vec>]) { + for y in xs { + for z in y { + if let Some(a) = z { k(a); } + } + } +} +fn le_d3(xs: &[Vec>], c: bool) { + for y in xs { + while c { + for z in y { + let Some(a) = z else { continue }; + k(a); + } + } + } +} +fn il_d3(xs: &[Vec>], c: bool) { + for y in xs { + while c { + for z in y { + if let Some(a) = z { k(a); } + } + } + } +} +fn le_two(xs: &[Option]) { + for y in xs { + let Some(a) = y else { continue }; + let Ok(b) = g(a) else { continue }; + k(b); + } +} +fn il_two(xs: &[Option]) { + for y in xs { + if let Some(a) = y { k(a); } + if let Ok(b) = g(*y) { k(b); } + } +} +fn le_cl() { + let c = |y: Option| { + let Some(a) = y else { return 0 }; + a + }; + k(c); +} +fn il_cl() { + let c = |y: Option| { + if let Some(a) = y { return a; } + 0 + }; + k(c); +} +fn le_in(x: Option, c: bool) -> i32 { + let Some(a) = x else { + if c { return 1; } + return 0; + }; + a +} +fn il_in(x: Option, c: bool) -> i32 { + if let Some(a) = x { + if c { return a; } + } + 0 +} +fn ie(x: bool) -> i32 { + if x { 1 } else { 0 } +} +"; + // (let-else fn, if-let twin, value, sum) + let expected: &[(&str, &str, u64, u64)] = &[ + ("le_lp", "il_lp", 3, 3), + ("le_top", "il_top", 1, 1), + ("le_d2", "il_d2", 6, 6), + ("le_d3", "il_d3", 10, 10), + ("le_two", "il_two", 5, 5), + ("le_cl", "il_cl", 0, 2), + ("le_in", "il_in", 3, 3), + ("ie", "ie", 2, 2), + ]; + check_func_space::(src, "foo.rs", |space| { + assert_eq!(space.spaces.len(), 15, "fixture members"); + let measured = |n: &str| { + let c = &child_space(&space, n).metrics.cognitive; + (c.cognitive(), c.cognitive_sum()) + }; + for &(name, twin, value, sum) in expected { + assert_eq!(measured(twin), (value, sum), "{twin}"); + assert_eq!(measured(name), (value, sum), "{name}"); + } + for name in ["le_cl", "il_cl"] { + let closure = &child_space(&space, name).spaces; + assert_eq!(closure.len(), 1, "{name}'s closure space"); + assert_eq!( + closure[0].metrics.cognitive.cognitive(), + 2, + "{name} closure" + ); + } + }); + } + #[cfg(feature = "typescript")] #[test] fn typescript_if_else_if_else() { @@ -7896,11 +8062,12 @@ end", // Companions to `php_null_coalescing_cognitive_230`: the PHP // cognitive operator set extends past `&&` / `||` / `??` to include - // the word-form `and` / `or` / `xor`, mirroring PHP cyclomatic. A - // chain of identical word-form operators collapses to a single + // the word-form `and` / `or`, mirroring PHP cyclomatic. A chain of + // identical word-form operators collapses to a single // boolean-sequence increment under Sonar B1, the same way `&&` / // `||` chains do. Each word-form gets its own test so a regression - // that drops a single variant (e.g. only `Or`) is still caught. + // that drops a single variant (e.g. only `Or`) is still caught; + // `xor` is a value, not a sequence (#1536), and its test pins that. #[cfg(feature = "php")] #[test] @@ -7942,21 +8109,27 @@ end", #[cfg(feature = "php")] #[test] - fn php_word_form_xor_forms_boolean_sequence_230() { - check_metrics::( - "( + &format!( + "); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct RunMember { + id: usize, + start: usize, + end: usize, + run_end: usize, +} + +impl ComparisonRuns { + /// The byte a boolean sequence under `node`, a `BinaryExpression3` + /// whose parent is `parent`, extends to: the end of the run `parent` + /// is a member of, else `node`'s own end. Records `node` when it is a + /// member itself. + fn sequence_end(&mut self, node: &Node, parent: Option) -> usize { + let (start, end) = (node.start_byte(), node.end_byte()); + while self + .0 + .last() + .is_some_and(|member| start < member.start || member.end < end) + { + self.0.pop(); + } + let run_end = match (parent, self.0.last()) { + (Some(parent), Some(member)) if member.id == parent.id() => member.run_end, + _ => end, + }; + if node.wraps_any(&RUN_COMPARISONS) { + self.0.push(RunMember { + id: node.id(), + start, + end, + run_end, + }); + } + run_end + } +} impl Cognitive for BashCode { fn compute<'a>( node: &Node<'a>, - _code: &'a [u8], + code: &'a [u8], ancestors: Ancestors<'a, '_>, stats: &mut Stats, nesting_map: &mut NestingMap, @@ -53,9 +134,19 @@ impl Cognitive for BashCode { // parenthesized sub-expressions). Verified empirically // against tree-sitter-bash 0.25.1 — the other four // `BinaryExpression*` enum variants never wrap `&&` / `||`. - List | BinaryExpression3 => { + // Inside `[ … ]` the same expression joins two tests with + // `-a` / `-o`, keyed to the symbol it stands for so the + // three spellings score alike (#1536). + List => { compute_booleans(node, stats, AMPAMP, PIPEPIPE); } + BinaryExpression3 => { + let end = stats.bash_runs.sequence_end(node, ancestors.parent(node)); + compute_booleans_by_node(node, end, stats, |child| match child.kind_id().into() { + AMPAMP | PIPEPIPE => Some(child.kind_id()), + _ => bash_test_connective(child, node, code).map(|op| op as u16), + }); + } FunctionDefinition => { enter_function_boundary(&mut nesting, node, ancestors, &[FunctionDefinition]); } diff --git a/src/metrics/cognitive/perl.rs b/src/metrics/cognitive/perl.rs index dff3882c..2243905a 100644 --- a/src/metrics/cognitive/perl.rs +++ b/src/metrics/cognitive/perl.rs @@ -12,6 +12,7 @@ )] use super::*; +use crate::lang_helpers::perl::perl_is_and_key_misparse; /// Folds a Perl `binary_expression`'s (or, for low-precedence `and`, /// `unary_expression`'s) short-circuit operator children @@ -92,8 +93,10 @@ impl Cognitive for PerlCode { // `unary_expression` (with `not`), while `or` / `xor` are a // `binary_expression`; without the second kind every `and` // scored nothing here while cyclomatic and ABC counted it - // (#1530). - P::BinaryExpression | P::UnaryExpression => { + // (#1530). FIXME(#1539 upstream): an auto-quoted `and` key + // (`(and => 1)`) is a `unary_expression` led by that token, + // and holds no operator. + P::BinaryExpression | P::UnaryExpression if !perl_is_and_key_misparse(node) => { compute_perl_booleans(node, stats); } P::FunctionDefinition | P::FunctionDefinitionWithoutSub => { diff --git a/src/metrics/cognitive/php.rs b/src/metrics/cognitive/php.rs index 04df6f56..d5d06459 100644 --- a/src/metrics/cognitive/php.rs +++ b/src/metrics/cognitive/php.rs @@ -62,16 +62,18 @@ impl Cognitive for PhpCode { } BinaryExpression => { // PHP's null-coalescing `??` short-circuits like `&&` / - // `||` and the word-form `and` / `or` / `xor`, so it - // forms boolean sequences alongside them. Mirrors the - // PHP cyclomatic operator set minus the assignment - // form `??=`, which is not a `BinaryExpression`. The word - // forms `and` / `or` are keyed to their symbols; `xor` - // and `??` are operations of their own (#1530). + // `||` and the word-form `and` / `or`, so it forms + // boolean sequences alongside them. Mirrors the PHP + // cyclomatic operator set minus the assignment form + // `??=`, which is not a `BinaryExpression`. The word + // forms `and` / `or` are keyed to their symbols; `??` is + // an operation of its own (#1530). `xor` evaluates both + // operands, so like `^` it is a value, not a sequence + // (#1536). compute_booleans_keyed(node, stats, |id| match id.into() { AMPAMP | And => Some(AMPAMP as u16), PIPEPIPE | Or => Some(PIPEPIPE as u16), - Xor | QMARKQMARK => Some(id), + QMARKQMARK => Some(id), _ => None, }); } diff --git a/src/metrics/cognitive/rust.rs b/src/metrics/cognitive/rust.rs index 8495a81d..2dc0b2f6 100644 --- a/src/metrics/cognitive/rust.rs +++ b/src/metrics/cognitive/rust.rs @@ -32,10 +32,20 @@ impl Cognitive for RustCode { ForExpression | WhileExpression | LoopExpression | MatchExpression => { increase_nesting(stats, &mut nesting); } + // `let PAT = e else { … };` is an `if let` that diverges + // instead of nesting the happy path, so it pays what its + // `if let` twin pays: a structural increment plus the current + // nesting, and a level for everything under it — the `else` + // block included, as the twin's body is (#1548). Keyed on the + // `alternative` field, as cyclomatic does (#1542). + LetDeclaration if node.child_by_field_name("alternative").is_some() => { + increase_nesting(stats, &mut nesting); + } // `Else` here is the `else` keyword token, which the grammar // also emits for the `else` of an `else if` — so this arm - // covers both. - Else => { + // covers both. A `let`-`else`'s token is a direct child of the + // `let_declaration`, which the arm above already charged. + Else if !ancestors.parent_has_kind(node, LetDeclaration as u16) => { increment_by_one(stats); } BreakExpression | ContinueExpression => { diff --git a/src/metrics/cognitive/word_operator_tests.rs b/src/metrics/cognitive/word_operator_tests.rs index 84ca2d66..2c62cc8a 100644 --- a/src/metrics/cognitive/word_operator_tests.rs +++ b/src/metrics/cognitive/word_operator_tests.rs @@ -11,8 +11,10 @@ //! //! The rows that switch operation (`a || b and c`) stay at two //! sequences: precedence decides which operator nests under which, and -//! the walk sees that through the parse. `xor`, `??` and `//` are -//! operations of their own and must not join an `&&` / `||` sequence. +//! the walk sees that through the parse. `??` and `//` are operations of +//! their own and must not join an `&&` / `||` sequence. `xor` evaluates +//! both operands, so it is a value like `^` and no sequence at all +//! (#1536). use std::cell::Cell; @@ -85,10 +87,10 @@ fn php_word_operators_continue_their_symbol_sequence() { ("$a && $b && $c", "$a and $b && $c", 1, 1), ("$a || $b || $c", "$a || $b or $c", 1, 1), ("($a || $b) && $c", "$a || $b and $c", 1, 2), - // `xor` is an operation of its own: it neither joins nor is - // joined by a conjunction. - ("$a ?? $b && $c", "$a xor $b && $c", 1, 2), - ("$a ?? $b ?? $c", "$a xor $b xor $c", 2, 1), + // `xor` is a value, as `^` is (#1536): it binds looser than + // `&&`, so only the conjunction is a sequence. + ("$a ^ ($b && $c)", "$a xor $b && $c", 1, 1), + ("$a ^ $b ^ $c", "$a xor $b xor $c", 2, 0), ]; assert_rows::("foo.php", asg, &words, &rows); } diff --git a/src/metrics/cyclomatic.rs b/src/metrics/cyclomatic.rs index 0cc59d9a..59012d3c 100644 --- a/src/metrics/cyclomatic.rs +++ b/src/metrics/cyclomatic.rs @@ -6084,22 +6084,172 @@ f() { } // `with` chains use `<-` arrows, which parse as `binary_operator` - // nodes — NOT `stab_clause`s — so the `with`-head clauses do not - // contribute to standard CCN per-arm. The fallthrough `else` - // branch, when present, contains `stab_clause`s that count for - // standard. The `with` Call itself is a multi-arm container Call - // that contributes once to modified CCN. + // nodes — NOT `stab_clause`s — and each is a decision in both tiers + // (#1535). The fallthrough `else` branch, when present, contains + // `stab_clause`s that count for standard, and collapses into one + // modified decision. #[cfg(feature = "elixir")] #[test] - fn elixir_with_else_only_counts_else_arms() { + fn elixir_with_counts_arrow_clauses_and_else_arms() { check_metrics::( "defmodule Foo do\n def f(x) do\n with {:ok, v} <- fetch(x),\n {:ok, w} <- fetch(v) do\n {:ok, w}\n else\n :error -> :nope\n other -> {:bad, other}\n end\n end\nend\n", "foo.ex", |metric| { - // standard: 3 entries + 2 else-block stabs = 5 - // modified: 3 entries + 1 with Call = 4 - assert_eq!(metric.cyclomatic.cyclomatic_sum(), 5); - assert_eq!(metric.cyclomatic.cyclomatic_modified_sum(), 4); + // standard: 3 entries + 2 `<-` clauses + 2 else-block + // stabs = 7 (#1535 added the `<-` clauses) + // modified: 3 entries + 2 `<-` clauses + 1 else + // dispatch = 6 + assert_eq!(metric.cyclomatic.cyclomatic_sum(), 7); + assert_eq!(metric.cyclomatic.cyclomatic_modified_sum(), 6); + }, + ); + } + + // A `with`'s `<-` clauses and a `for … reduce:`'s head clause + // (#1535). Each row is one `def`, read as (cyclomatic, modified, + // abc.conditions); the function's base path is the 1. + // + // Expected values, from the twins rather than the fix (measured, + // `--no-config`, as cyclomatic / modified / ABC conditions): + // - `w1`: one `<-` is one decision, Rust's + // `if let Ok(a) = g(x) { a }` — 2/2/1, and `g(x)?` is 2/2/0. + // - `w2` / `w2d`: two `<-` are two, Rust's nested `if let` — 3/3/2. + // `w2d`'s `else _ -> 0` picks between nothing, so it is free in + // both tiers, as Rust's `else { 0 }` is. + // - `w1e`: the `else {:error, e} ->` arm pays per clause as a + // `case` arm does, and its dispatch collapses into one modified + // decision: 1 + `<-` + arm = 3, and 3 modified. `w1n`'s named + // `e ->` binds, so it is no bare `_` and scores the same. `w1g`'s + // guarded `_ when x > 1 ->` is a decision with a guard on it: + // 1 + `<-` + clause + guard = 4, and modified collapses only the + // clause — 4. + // - `w2e`: 1 + two `<-` + two arms = 5; modified 1 + 2 + 1 = 4. + // - `wg`: the guard is one more decision on the `<-`, Rust's + // `if let Ok(a) = g(x) && a > 0` — 3/3/2. + // - `wb`: a bare match `b = …` in a `with` cannot fail over to the + // `else`, so it is no clause — 2/2/1, `w1`'s value. + // - `wk` / `wke`: the keyword form scores as the block form does. + // - `r1` / `rk1`: a single reduce clause always matches, so the + // reduce scores what `fp`, the `for` without one, does — 2/2/0. + // - `r2` / `r2d`: two reduce clauses are one decision, n−1 as a + // multi-clause `fn` is (#776), the trailing `_ ->` included — + // 3/2/1. `rk2` is `r2` in the keyword form. + // - `fp` / `ff` / `fg` / `fb`: a `for` generator's `<-` is not a + // `with` clause, nor is a guard on it (`fg` scores as the bare + // filter `ff`), nor a bitstring generator's — each scores the + // `for`'s one decision, as before #1535. + // - `c`: `case {:ok, a} / _`, the Rust `match` twin — 2/2/1. + // + // ABC conditions sit at `cyclomatic - 1` on every row but the + // `for`s, whose loop decision ABC deliberately does not count. + #[cfg(feature = "elixir")] + #[test] + fn elixir_with_arrow_clauses_and_reduce_head_1535() { + let src = "defmodule M do + def w1(x) do + with {:ok, a} <- g(x) do a end + end + def w2(x) do + with {:ok, a} <- g(x), {:ok, b} <- h(a) do b end + end + def w2d(x) do + with {:ok, a} <- g(x), {:ok, b} <- h(a) do b else _ -> 0 end + end + def w1e(x) do + with {:ok, a} <- g(x) do a else {:error, e} -> e end + end + def w1n(x) do + with {:ok, a} <- g(x) do a else e -> e end + end + def w1g(x) do + with {:ok, a} <- g(x) do a else _ when x > 1 -> 0 end + end + def w2e(x) do + with {:ok, a} <- g(x), {:ok, b} <- h(a) do + b + else + {:error, e} -> e + :nope -> 0 + end + end + def wg(x) do + with {:ok, a} when is_integer(a) <- g(x) do a end + end + def wb(x) do + with {:ok, a} <- g(x), b = a + 1 do b end + end + def wk(x), do: with({:ok, a} <- g(x), do: a, else: (_ -> 0)) + def wke(x), do: with({:ok, a} <- g(x), do: a, else: ({:error, e} -> e)) + def r1(xs) do + for x <- xs, reduce: 0 do + acc -> acc + x + end + end + def rk1(xs), do: for(x <- xs, reduce: 0, do: (acc -> acc + x)) + def r2(xs) do + for x <- xs, reduce: 0 do + 0 -> x + acc -> acc + x + end + end + def r2d(xs) do + for x <- xs, reduce: 0 do + 0 -> x + _ -> 1 + end + end + def rk2(xs), do: for(x <- xs, reduce: 0, do: (0 -> x; acc -> acc + x)) + def fp(xs), do: for(x <- xs, do: x) + def ff(xs), do: for(x <- xs, x > 1, do: x) + def fg(xs), do: for(x when x > 1 <- xs, do: x) + def fb(s), do: for(<>, do: c) + def c(x) do + case g(x) do + {:ok, a} -> a + _ -> 0 + end + end +end +"; + let expected: &[(&str, u64, u64, u64)] = &[ + ("w1", 2, 2, 1), + ("w2", 3, 3, 2), + ("w2d", 3, 3, 2), + ("w1e", 3, 3, 2), + ("w1n", 3, 3, 2), + ("w1g", 4, 4, 3), + ("w2e", 5, 4, 4), + ("wg", 3, 3, 2), + ("wb", 2, 2, 1), + ("wk", 2, 2, 1), + ("wke", 3, 3, 2), + ("r1", 2, 2, 0), + ("rk1", 2, 2, 0), + ("r2", 3, 2, 1), + ("r2d", 3, 2, 1), + ("rk2", 3, 2, 1), + ("fp", 2, 2, 0), + ("ff", 2, 2, 1), + ("fg", 2, 2, 1), + ("fb", 2, 2, 0), + ("c", 2, 2, 1), + ]; + check_func_space_only::( + src, + "foo.ex", + &[Metric::Cyclomatic, Metric::Abc], + |space| { + let module = &space.spaces[0]; + assert_eq!(module.spaces.len(), expected.len(), "fixture members"); + for &(name, cyclomatic, modified, conditions) in expected { + let f = child_space(module, name); + let measured = ( + f.metrics.cyclomatic.cyclomatic(), + f.metrics.cyclomatic.cyclomatic_modified(), + f.metrics.abc.conditions(), + ); + assert_eq!(measured, (cyclomatic, modified, conditions), "{name}"); + } }, ); } @@ -7223,6 +7373,99 @@ f() { ); } + // A `let`-`else` binds or diverges: one decision (#1542). Each row is + // one `fn`, read as (cyclomatic, modified, abc.conditions). + // + // Expected values come from the twins, not the fix (measured, + // `--no-config`): `il` (`if let`) and `m` (`match` with a bare `_`) + // both score 2/2/1, so `le` must as well. `ctrl` is the same body + // without the `else`, at 1/1/0. `le2` pays once per `let`-`else`, 3/3/2. + // `ie` pins that the shared `else` token of an `if … else` still pays + // nothing extra (2/2, with ABC's slot for `x` and the `else`). `lp` + // nests one in a `for`: 1 + `for` + `let`-`else` = 3, where ABC counts + // only the `else`, because it does not score loops. `cl`'s closure + // opens its own space, which pays the decision itself. + #[cfg(feature = "rust")] + #[test] + fn rust_let_else_is_one_decision_1542() { + let src = "fn le(x: i32) -> i32 { + let Ok(a) = g(x) else { return 0 }; + a +} +fn ctrl(x: i32) -> i32 { + let a = g(x); + a +} +fn il(x: i32) -> i32 { + if let Ok(a) = g(x) { return a; } + 0 +} +fn m(x: i32) -> i32 { + match g(x) { Ok(a) => a, _ => 0 } +} +fn le2(x: i32) -> i32 { + let Ok(a) = g(x) else { return 0 }; + let Some(b) = h(a) else { return 1 }; + b +} +fn ie(x: bool) -> i32 { + if x { 1 } else { 0 } +} +fn lp(xs: &[Option]) { + for y in xs { + let Some(a) = y else { continue }; + k(a); + } +} +fn cl() { + let c = |y: Option| { + let Some(a) = y else { return 0 }; + a + }; + k(c); +} +"; + let expected: &[(&str, u64, u64, u64)] = &[ + ("le", 2, 2, 1), + ("ctrl", 1, 1, 0), + ("il", 2, 2, 1), + ("m", 2, 2, 1), + ("le2", 3, 3, 2), + ("ie", 2, 2, 2), + ("lp", 3, 3, 1), + ("cl", 1, 1, 0), + ]; + check_func_space_only::( + src, + "foo.rs", + &[Metric::Cyclomatic, Metric::Abc], + |space| { + assert_eq!(space.spaces.len(), expected.len(), "fixture members"); + for &(name, cyclomatic, modified, conditions) in expected { + let f = child_space(&space, name); + let measured = ( + f.metrics.cyclomatic.cyclomatic(), + f.metrics.cyclomatic.cyclomatic_modified(), + f.metrics.abc.conditions(), + ); + assert_eq!(measured, (cyclomatic, modified, conditions), "{name}"); + } + let closure = child_space(&space, "cl"); + assert_eq!(closure.spaces.len(), 1, "cl's closure space"); + let closure = &closure.spaces[0].metrics; + assert_eq!( + ( + closure.cyclomatic.cyclomatic(), + closure.cyclomatic.cyclomatic_modified(), + closure.abc.conditions(), + ), + (2, 2, 1), + "cl's closure" + ); + }, + ); + } + /// Regression #107: empty case…esac has no arms, so standard adds 0 and /// modified adds 1 (the container). #[cfg(feature = "bash")] diff --git a/src/metrics/cyclomatic/bash.rs b/src/metrics/cyclomatic/bash.rs index ebfeda70..3fc84bad 100644 --- a/src/metrics/cyclomatic/bash.rs +++ b/src/metrics/cyclomatic/bash.rs @@ -7,12 +7,13 @@ )] use super::*; +use crate::lang_helpers::bash::bash_test_connective; impl Cyclomatic for BashCode { fn compute<'a>( node: &Node<'a>, code: &'a [u8], - _ancestors: Ancestors<'a, '_>, + ancestors: Ancestors<'a, '_>, stats: &mut Stats, ) { match node.kind_id().into() { @@ -51,6 +52,16 @@ impl Cyclomatic for BashCode { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } + // `[ … ]` spells `&&` / `||` as `-a` / `-o`, a decision each + // like the `[ … ] && [ … ]` and `[[ … && … ]]` twins (#1536). + Bash::TestOperator + if ancestors + .parent(node) + .is_some_and(|parent| bash_test_connective(node, &parent, code).is_some()) => + { + stats.cyclomatic += 1.; + stats.cyclomatic_modified += 1.; + } _ => {} } } diff --git a/src/metrics/cyclomatic/elixir.rs b/src/metrics/cyclomatic/elixir.rs index 88eb693b..30793877 100644 --- a/src/metrics/cyclomatic/elixir.rs +++ b/src/metrics/cyclomatic/elixir.rs @@ -7,6 +7,10 @@ )] use super::*; +use crate::metrics::elixir_decisions::{ + elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_when_is_guard, + elixir_with_else_dispatches, +}; impl Cyclomatic for ElixirCode { // Elixir's control-flow constructs are not distinct grammar @@ -19,22 +23,25 @@ impl Cyclomatic for ElixirCode { // The split between standard and modified CCN mirrors the C-family // case/switch treatment: per-arm `stab_clause` nodes contribute // standard, while the multi-arm container Calls (`case`/`cond`/ - // `with`/`try`) contribute modified. Single-branch keyword Calls - // (`if`/`unless`/`for`/`while`) contribute to both. Short-circuit - // booleans (`&&`, `||`, `and`, `or`) contribute to both. + // `try`) contribute modified. Single-branch keyword Calls + // (`if`/`unless`/`for`/`while`) contribute to both, as do a `with`'s + // `<-` clauses and short-circuit booleans (`&&`, `||`, `and`, `or`). fn compute<'a>( node: &Node<'a>, code: &'a [u8], ancestors: Ancestors<'a, '_>, stats: &mut Stats, ) { + // bca: suppress(cyclomatic, halstead) — kind dispatch table; each gated arm is one rule use Elixir as E; match node.kind_id().into() { // Per-arm decisions: each `stab_clause` is one arm of a // `case`/`cond`/`with`/anonymous-fn body or a `rescue`/ - // `catch` handler. Standard-only — modified counts the - // container Call once. + // `catch` handler. Standard-only: modified counts a + // `case`/`cond`/`try` container Call once, a `with` once + // only when its `else` dispatches (the `"with"` arm below), + // and gives a multi-clause `fn` or `reduce` nothing. // // The exception is the *first* `stab_clause` of an // `anonymous_function` (`fn … -> … end`): it is the @@ -44,9 +51,9 @@ impl Cyclomatic for ElixirCode { // `getter::elixir` → `SpaceKind::Function`), so counting the // head clause too over-reports a trivial `fn x -> x end` as // 2 (issue #776). Only the 2nd+ clauses of a multi-clause - // `fn` are real branches. `case`/`cond`/`with` arms have a - // `do_block` parent — not `anonymous_function` — so they are - // unaffected and keep counting. + // `fn` are real branches. A `for … reduce:`'s accumulator + // clauses take the same skip (#1535): a single `acc -> …` + // always matches. `case`/`cond`/`with` arms keep counting. // // Two further clause shapes are the construct's default arm // and are excluded to match the sibling family — Rust's @@ -69,9 +76,7 @@ impl Cyclomatic for ElixirCode { // `case`. Guarded forms (`_ when g ->`, `true when g ->`) // and named discards (`_x ->`) are real decisions and // still count. - E::StabClause - if crate::metrics::npa::elixir_clause_is_decision(node, code, ancestors) => - { + E::StabClause if elixir_clause_is_decision(node, code, ancestors) => { stats.cyclomatic += 1.; } // A guard is a decision the construct it guards does not @@ -95,13 +100,36 @@ impl Cyclomatic for ElixirCode { // No double count (§5): the token fires once per `when`, // and the `binary_operator` that wraps it is not matched by // any arm here. - E::When if crate::metrics::npa::elixir_when_is_guard(node, code, ancestors) => { + E::When if elixir_when_is_guard(node, code, ancestors) => { + stats.cyclomatic += 1.; + stats.cyclomatic_modified += 1.; + } + // A `with`'s `pattern <- expr` clause either matches or + // short-circuits to the `else`, so each is a decision + // (#1535) — the Rust `if let` / `?` twin pays one in both + // tiers. The clauses run in sequence rather than being + // alternatives of one dispatch, so modified does not + // collapse them. A `for` generator's `<-` is excluded by + // the gate: the `for` already pays its loop decision. + E::LTDASH if elixir_arrow_is_with_clause(node, code, ancestors) => { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } // Short-circuit booleans add a decision point in both - // metrics. - E::AMPAMP | E::PIPEPIPE | E::And | E::Or => { + // metrics — where applied. `&and/2` and `&||/2` only name the + // operator and decide nothing (#1534); ABC and cognitive gate + // on the same parent. A remote call naming it, + // `Kernel.||(a, b)`, applies the macro and decides like + // `a || b`; a user module's `Foo.||(a, b)` is an ordinary + // call and does not. ABC and cognitive still score the + // `Kernel` call 0, as they did before #1534. + E::AMPAMP | E::PIPEPIPE | E::And | E::Or + if crate::lang_helpers::elixir::elixir_applying_operator(node, ancestors) + .is_some() + || crate::lang_helpers::elixir::elixir_remote_call_applies( + node, code, ancestors, + ) => + { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } @@ -123,7 +151,15 @@ impl Cyclomatic for ElixirCode { // (the container collapses to a single decision). // Per-arm `stab_clause`s already contribute to // standard above. - "case" | "cond" | "with" | "try" => { + "case" | "cond" | "try" => { + stats.cyclomatic_modified += 1.; + } + // A `with`'s decisions are its `<-` clauses, + // counted above in both tiers. What modified + // collapses is its `else`, the dispatch over + // the failed value — once, and only when that + // dispatch picks between anything (#1535). + "with" if elixir_with_else_dispatches(node, code) => { stats.cyclomatic_modified += 1.; } _ => {} diff --git a/src/metrics/cyclomatic/perl.rs b/src/metrics/cyclomatic/perl.rs index d6fd8c6c..68c9e0a4 100644 --- a/src/metrics/cyclomatic/perl.rs +++ b/src/metrics/cyclomatic/perl.rs @@ -7,12 +7,13 @@ )] use super::*; +use crate::lang_helpers::perl::perl_and_is_operator; impl Cyclomatic for PerlCode { fn compute<'a>( node: &Node<'a>, _code: &'a [u8], - _ancestors: Ancestors<'a, '_>, + ancestors: Ancestors<'a, '_>, stats: &mut Stats, ) { use Perl as P; @@ -41,12 +42,22 @@ impl Cyclomatic for PerlCode { | P::AMPAMPEQ | P::PIPEPIPEEQ | P::SLASHSLASHEQ - | P::And | P::Or | P::TernaryExpression => { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } + // FIXME(#1539 upstream): an auto-quoted `and` key lexes as + // the operator token; only one with a left operand is a + // decision. + P::And + if ancestors + .parent(node) + .is_some_and(|parent| perl_and_is_operator(node, &parent)) => + { + stats.cyclomatic += 1.; + stats.cyclomatic_modified += 1.; + } _ => {} } } diff --git a/src/metrics/cyclomatic/php.rs b/src/metrics/cyclomatic/php.rs index b769ace3..78dc250f 100644 --- a/src/metrics/cyclomatic/php.rs +++ b/src/metrics/cyclomatic/php.rs @@ -27,7 +27,9 @@ impl Cyclomatic for PhpCode { SwitchStatement | MatchExpression => { stats.cyclomatic_modified += 1.; } - // Both standard and modified. + // Both standard and modified. `xor` is absent: it evaluates + // both operands, so like `^` it is a value, not a decision + // (#1536). IfStatement | ElseIfClause | ElseIfClause2 @@ -41,7 +43,6 @@ impl Cyclomatic for PhpCode { | PIPEPIPE | And | Or - | Xor | QMARKQMARK | QMARKQMARKEQ // Nullsafe operator `?->` (`QMARKDASHGT`) is short-circuit — it diff --git a/src/metrics/cyclomatic/rust.rs b/src/metrics/cyclomatic/rust.rs index 03a7666f..e5eedbd1 100644 --- a/src/metrics/cyclomatic/rust.rs +++ b/src/metrics/cyclomatic/rust.rs @@ -80,6 +80,14 @@ fn rust_cyclomatic_increment(node: &Node<'_>, stats: &mut Stats, count_try: bool stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } + // `let PAT = e else { … };` either binds or diverges: one + // decision, as its `if let` twin pays. Keyed on the + // `alternative` field rather than the shared `else` token, which + // `if … else` also emits. + LetDeclaration if node.child_by_field_name("alternative").is_some() => { + stats.cyclomatic += 1.; + stats.cyclomatic_modified += 1.; + } // Both standard and modified. If | For | While | Loop | AMPAMP | PIPEPIPE => { stats.cyclomatic += 1.; diff --git a/src/metrics/elixir_decisions.rs b/src/metrics/elixir_decisions.rs new file mode 100644 index 00000000..03f891fa --- /dev/null +++ b/src/metrics/elixir_decisions.rs @@ -0,0 +1,469 @@ +//! Elixir clause and guard predicates shared by the `Cyclomatic` and +//! `Abc` impls for `ElixirCode`. +//! +//! Elixir spells every control-flow construct as a `Call`, so which +//! `stab_clause`, `when` and `<-` nodes are decisions is a question of +//! position and source text rather than node kind. Both metrics must +//! answer it identically (grammar-dispatch §7), so the answer lives in +//! one module. + +use crate::lang_helpers::elixir::{elixir_call_keyword, elixir_is_method_macro}; +use crate::{Ancestors, Elixir, Node}; + +/// Whether `node` is a `binary_operator` applying `operator`. +/// +/// `binary_operator` carries three kind aliases at this pin so it is +/// matched by rule name rather than by enumerating ids +/// (grammar-dispatch §1); `when` and `<-` each have exactly one id. +fn elixir_is_binary_operator(node: &Node, operator: Elixir) -> bool { + const BINARY_OPERATOR: &str = "binary_operator"; + + node.kind() == BINARY_OPERATOR + && node + .child_by_field_name("operator") + .is_some_and(|token| token.kind_id() == operator as u16) +} + +/// Whether `node` is a `when` `binary_operator` — the shape a repeated +/// guard's alternatives nest through. +fn elixir_is_when_operator(node: &Node) -> bool { + elixir_is_binary_operator(node, Elixir::When) +} + +/// Whether the two ancestors `up` yields first are a `with` Call's +/// `arguments` and the Call itself — the position of a `with` clause. +/// +/// `arguments` carries five kind aliases at this pin, so it is matched +/// by rule name (grammar-dispatch §1). +fn elixir_is_with_argument<'a>(mut up: impl Iterator>, code: &'a [u8]) -> bool { + up.next() + .is_some_and(|arguments| arguments.kind() == "arguments") + && up + .next() + .is_some_and(|call| elixir_call_keyword(&call, code) == Some("with")) +} + +/// Whether `node`, a `<-` token, is the operator of one of a `with`'s +/// `pattern <- expr` clauses: a decision, since the value either +/// matches the pattern or short-circuits to the `else` clauses (#1535). +/// It is the shape a Rust `if let` or `?` has, both of which pay one +/// decision in standard and modified cyclomatic alike. +/// +/// Only a direct argument of a `with` qualifies. The same token is a +/// `for` comprehension's generator (`for x <- xs`), which filters +/// rather than short-circuits and is deliberately left alone — `for` +/// already pays one decision for its loop — and a bitstring +/// generator (`for <>`) sits inside a `bitstring`, not under +/// `arguments`. +/// +/// Position alone is the gate, so it needs no separate test that the +/// token is applied (#1534): in every shape probed where it is not — +/// named (`with &<-/2`) or stranded by error recovery (`with(<-)`, +/// `with a, <- b`) — it sits under an `operator_identifier`, whose +/// parent is never the `with`'s `arguments`. +/// +/// A clause wrapped in parentheses after a space (`with (a <- b) do`) +/// parses through a `block` and is not recognised; Elixir drops the +/// parentheses, so it is a clause, but the spelling is rare. +/// +/// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode` +/// (grammar-dispatch §7). +pub(super) fn elixir_arrow_is_with_clause<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> bool { + elixir_is_with_argument(ancestors.iter(node).skip(1).map(|(a, _)| a), code) +} + +/// The one alternative a guard's `when` token introduces. +/// +/// `head when a when b` parses right-associatively as +/// `head when (a when b)`, so each `when` operator owns exactly one +/// alternative: the nested operator's `left` where its `right` nests a +/// further `when`, and its own `right` otherwise. One alternative per +/// token is what holds the `Abc` slot count level with the `Cyclomatic` +/// decision count at every chain length (§8), and what stops a chain of +/// n alternatives being counted once per token that can see it (§5). +/// +/// The `?` is infallible at the pinned grammar — `binary_operator` +/// declares `right` required — and is spelled as an `Option` because +/// `AGENTS.md` bans `expect` outside tests. +pub(super) fn elixir_when_alternative<'a>(when_operator: &Node<'a>) -> Option> { + let right = when_operator.child_by_field_name("right")?; + if elixir_is_when_operator(&right) { + right.child_by_field_name("left") + } else { + Some(right) + } +} + +/// Whether a `when` operator token spells a real guard — a function +/// head's (`def f(x) when g do`) or a clause's (`x when g -> …`) — +/// rather than a typespec's `when` binding clause +/// (`@spec f(a) :: a when a: integer`), which is type syntax and no +/// decision at all. +/// +/// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode` so the +/// two cannot disagree about what a guard is (grammar-dispatch §7). +/// Elixir has no dedicated guard production — `x when g` is an ordinary +/// `binary_operator` — so the position it sits in is the only thing that +/// tells a guard from a typespec, and the allowlist below is that +/// position set at the pinned grammar: the `left` slot of a +/// `stab_clause` (`case` / `cond` / `fn` / `receive` / `with`'s `else` +/// / `try`'s handlers), the pattern of a `with` clause +/// (`{:ok, a} when is_integer(a) <- g(x)`, #1535), or an argument of a +/// definition Call that takes a guarded head. +/// +/// `arguments` carries five kind aliases at this pin and +/// `binary_operator` three, so both are matched by rule name rather +/// than by enumerating ids (grammar-dispatch §1). +/// +/// Repeated guards (`when a when b`) are an or-chain: Elixir tries each +/// `when` expression in turn and moves to the next when the previous one +/// is false *or raises*, so the construct carries one decision per +/// alternative, level with the `when a or b` spelling. They parse +/// right-associatively into nested `when` operators, and only the +/// outermost sits on the anchor — so a nested one is a guard too, and +/// the climb below walks the `when` operators between it and the anchor +/// before asking the position question. [`elixir_when_alternative`] +/// names the one alternative each token introduces, which is how the +/// matching `Abc` slot stays one-per-token rather than double counting +/// the chain (grammar-dispatch §5). +/// +/// The climb stops at the first non-`when` ancestor, so an ordinary +/// single guard pays no extra step and a chain pays one per alternative +/// it lists — a bound set by the guard, not by the tree's depth. +/// +/// Both `None`s the climb answers `false` to are unreachable at the +/// pin: a `when` *token*'s chain always holds at least the +/// `binary_operator` it belongs to, and that operator always sits under +/// something, because the Elixir root is `source` and no `when` +/// operator can be it. They are `is_some_and`s rather than `expect`s +/// because `AGENTS.md` bans the `expect` outside tests. +pub(super) fn elixir_when_is_guard<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> bool { + let mut chain = ancestors.iter(node).map(|(ancestor, _)| ancestor); + // The token's parent is the `when` operator node itself; the first + // ancestor above the chain of `when` operators is the position that + // decides. + chain.next().is_some_and(|mut operator| { + let mut above = chain.next(); + while let Some(enclosing) = above.filter(elixir_is_when_operator) { + operator = enclosing; + above = chain.next(); + } + above.is_some_and(|parent| elixir_is_guard_position(&operator, &parent, chain, code)) + }) +} + +/// Whether `parent` — the first ancestor above the chain of `when` +/// operators that `operator` tops — holds that chain in a guard +/// position; `up` yields the ancestors above `parent`. The second step +/// of [`elixir_when_is_guard`], after the climb. +fn elixir_is_guard_position<'a>( + operator: &Node<'a>, + parent: &Node<'a>, + mut up: impl Iterator>, + code: &'a [u8], +) -> bool { + use Elixir as E; + + const ARGUMENTS: &str = "arguments"; + + if parent.kind_id() == E::StabClause as u16 { + return parent + .child_by_field_name("left") + .is_some_and(|left| left.id() == operator.id()); + } + // A guard on the pattern of a `with` clause. `when` binds tighter + // than `<-`, so it parses as the clause's `left`; a `when` on the + // right would be a guard outside any guard position, which Elixir + // rejects, so the side is not checked. The guard of a `for` + // generator (`for x when x > 1 <- xs`) is a filter, which the + // comprehension's own decision already covers exactly as a bare + // filter (`for x <- xs, x > 1`) is — so it is not a guard here. + if elixir_is_binary_operator(parent, E::LTDASH) { + return elixir_is_with_argument(up, code); + } + parent.kind() == ARGUMENTS + && up.next().is_some_and(|call| { + elixir_call_keyword(&call, code).is_some_and(|keyword| { + elixir_is_method_macro(keyword) || matches!(keyword, "defguard" | "defguardp") + }) + }) +} + +/// Returns the sole pattern of a `stab_clause` whose left-hand side +/// carries no `when` guard, or `None` otherwise. +/// +/// The grammar's `left` field is an `arguments` node for a plain +/// pattern list, but a guarded clause (`_ when g ->`) re-shapes it +/// into a `binary_operator` wrapping the patterns and the guard — so +/// checking the field's kind answers "is there a guard" for free. The +/// kind is compared as a string because the grammar aliases several +/// internal rules to `arguments` with distinct kind ids +/// (`Arguments2`..`Arguments5`); grammar-dispatch §1 prefers the one +/// string comparison over enumerating them. A zero-arity clause carries +/// no `left` field at all (`fn -> … end`), which the same `filter` folds +/// into "no plain pattern list" — the two spellings of "nothing to +/// inspect" are one question, so they share one exit. Named-children +/// filtering skips the anonymous `(` `)` tokens a parenthesised clause +/// head carries, so an empty pair (`fn () -> … end`) also yields zero +/// patterns. Clauses with zero or several patterns return `None`. +pub(super) fn elixir_sole_unguarded_pattern<'a>(node: &Node<'a>) -> Option> { + let left = node + .child_by_field_name("left") + .filter(|left| left.kind() == "arguments")?; + let mut patterns = left.children().filter(Node::is_named); + let sole = patterns.next()?; + patterns.next().is_none().then_some(sole) +} + +/// Whether `pattern` is a bare `_`. Elixir has no wildcard token — `_` +/// parses as an ordinary `identifier` — so the bytes decide. +fn elixir_is_bare_wildcard(pattern: &Node, code: &[u8]) -> bool { + pattern.kind_id() == Elixir::Identifier as u16 && pattern.utf8_text(code) == Some("_") +} + +/// Returns `true` when `node` is a construct's free default clause: +/// a bare `_ ->` catch-all outside a head-skipping construct (an +/// anonymous fn or a `for … reduce:`), or an unguarded `true ->` +/// directly under a `cond`. +/// +/// Both shapes hinge on the clause's sole unguarded pattern, and +/// [`elixir_sole_unguarded_pattern`]'s child scan allocates a cursor +/// (the #1112 malloc) — so the pattern is extracted once here and the +/// two shapes branch on its (kind, text), instead of every counted +/// `stab_clause` paying the extraction twice through two independent +/// predicates. +/// +/// Bare `_ ->`: Elixir has no dedicated wildcard token — `_` parses +/// as an ordinary `identifier` — so the bytes decide (grammar-dispatch +/// §10). A named discard (`_x ->`) binds a value the body can read +/// and keeps counting, matching Rust's bare-`_`-only `MatchArm` rule; +/// guarded wildcards never reach the text check because +/// [`elixir_sole_unguarded_pattern`] rejects them. The exclusion does +/// NOT apply when `skips_head` — the clause's container is an +/// `anonymous_function` or a `for … reduce:`: such a multi-clause +/// dispatch is like `case` — n clauses are n−1 decisions — and its free +/// base path is already granted by the head-clause skip (#776, #1535), +/// so excluding a trailing `_ ->` too would leave +/// `fn 0 -> :a; _ -> :b end` at zero decisions while the identical +/// `case` reports one. +/// +/// `true ->` under `cond`: the exclusion is *shape-based* — any +/// unguarded `true ->` whose parent is the `do_block` of a `Call` +/// spelling `cond`, whatever the arm's position. That deliberately +/// matches the sibling convention: Rust's bare-`_` `MatchArm` +/// exclusion is equally position-blind. The same clause under `case` +/// is an ordinary boolean pattern match, so the exclusion is anchored +/// to the owning construct (grammar-dispatch §8); a guarded +/// `true when g ->` is a real decision and never reaches the +/// container check. +fn elixir_is_default_clause<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, + skips_head: bool, +) -> bool { + use Elixir as E; + + let Some(pattern) = elixir_sole_unguarded_pattern(node) else { + return false; + }; + match pattern.kind_id().into() { + // Bare `_` catch-all — free everywhere except where the + // head-clause skip already provides the free base path. + E::Identifier => !skips_head && elixir_is_bare_wildcard(&pattern, code), + // Unguarded `true` — free only as `cond`'s designated + // default; the O(1) parent/grandparent checks run after the + // text compare so non-`true` booleans bail early. + E::Boolean => { + pattern.utf8_text(code) == Some("true") + && elixir_do_section_call(node, code, ancestors) + .is_some_and(|call| elixir_call_keyword(&call, code) == Some("cond")) + } + _ => false, + } +} + +/// Whether `node`'s clauses-owner grants a free head clause: an +/// `anonymous_function` (#776), or a `for … reduce:` comprehension +/// (#1535), whose accumulator clauses are an anonymous fn in all but +/// spelling — a single `acc -> …` always matches, and only the 2nd+ +/// clauses dispatch. +/// +/// A `for` holds clauses in its `do` section only under `reduce:`; +/// without it the section is a plain body. So the Call's keyword alone +/// identifies the shape, with no need to find the `reduce:` pair. +fn elixir_clauses_skip_head<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> bool { + ancestors.parent_has_kind(node, Elixir::AnonymousFunction as u16) + || elixir_do_section_call(node, code, ancestors) + .is_some_and(|call| elixir_call_keyword(&call, code) == Some("for")) +} + +/// Returns `true` when `node` is its parent's first `stab_clause` — +/// the head clause of a construct [`elixir_clauses_skip_head`] names. +/// +/// The parent is the `anonymous_function` (`fn stab_clause+ end`), the +/// `do_block` (`do stab_clause+ end`) or the keyword form's +/// parenthesised `block`; in each the first child whose kind is +/// `stab_clause` (skipping the `fn` / `do` / `(` token) is the head. +fn elixir_is_first_clause<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + ancestors + .parent(node) + .and_then(|parent| { + parent + .children() + .find(|child| child.kind_id() == Elixir::StabClause as u16) + }) + .is_some_and(|first| first.id() == node.id()) +} + +/// Whether `node` (a `stab_clause`) is one decision of the construct +/// that owns it: every clause of a `case` / `cond` / `with`'s `else` / +/// `receive` / `try` handler and every clause after a multi-clause +/// `fn`'s first, except the construct's free default clause +/// ([`elixir_is_default_clause`]). +/// +/// An anonymous fn's head clause is the closure's definition, not a +/// dispatch: the closure opens its own function space whose base path +/// already covers it, so counting it too would score a trivial +/// `fn x -> x end` as a decision (#776). A `for … reduce:`'s first +/// accumulator clause is skipped for the same reason: the `for` already +/// pays its one decision, and a single `acc -> …` clause always matches +/// (#1535). +/// +/// The parent is allowlisted because the grammar puts a `stab_clause` +/// in one more position: a parenthesised `block`, which is how a +/// typespec spells a function type (`@spec f((any -> any)) :: list`). +/// That is type syntax and branches on nothing, the same reason +/// [`elixir_when_is_guard`] excludes a typespec's `when` (#1531). The +/// keyword form of a clause construct (`case(x, do: (1 -> :a; …))`) +/// holds its clauses in the same parenthesised `block`, so a `block` +/// counts when it is the value of a section pair +/// ([`elixir_is_section_pair`]). +/// +/// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode`, which +/// both score a clause construct per clause and must agree on which +/// clauses those are (grammar-dispatch §7, #1531). +pub(super) fn elixir_clause_is_decision<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> bool { + use Elixir as E; + + let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); + let in_section = up.next().is_some_and(|parent| { + matches!( + parent.kind_id().into(), + E::DoBlock + | E::ElseBlock + | E::AfterBlock + | E::RescueBlock + | E::CatchBlock + | E::AnonymousFunction + ) || (parent.kind_id() == E::Block as u16 + && up + .next() + .is_some_and(|pair| elixir_is_section_pair(&pair, code, SECTION_KEYS))) + }); + if !in_section { + return false; + } + let skips_head = elixir_clauses_skip_head(node, code, ancestors); + if skips_head && elixir_is_first_clause(node, ancestors) { + return false; + } + !elixir_is_default_clause(node, code, ancestors, skips_head) +} + +/// Whether a `with` Call's `else` clauses dispatch: whether at least one +/// of them is a decision rather than the free bare `_ ->` default. +/// +/// Modified cyclomatic collapses a clause construct's arms into one +/// decision, which for `case` / `cond` / `try` is the container paying +/// once. A `with`'s decisions are its `<-` clauses, each paid on its own +/// in both tiers as a Rust `if let` is; its `else` is the dispatch over +/// the failed value, which collapses into one decision exactly when it +/// holds one — so `else _ -> 0`, which picks nothing, stays free in +/// modified as it is in standard, like Rust's `if let … else` (#1535). +/// Spelled in either form: the block form's `else_block`, or the +/// keyword form's `else:` pair whose value is a parenthesised `block`. +/// +/// In a `with`'s `else` the default-clause rule reduces to the bare +/// `_`: `true ->` is free only under a `cond`, and the head skip only +/// under a `fn` or `reduce:`, so [`elixir_clause_is_decision`] agrees +/// clause by clause with the test below. +pub(super) fn elixir_with_else_dispatches(call: &Node, code: &[u8]) -> bool { + use Elixir as E; + + call.children().any(|child| { + let section = match child.kind_id().into() { + E::DoBlock => child + .children() + .find(|c| c.kind_id() == E::ElseBlock as u16), + _ if child.kind() == "arguments" => child + .children() + .filter(|c| c.kind_id() == E::Keywords as u16) + .flat_map(|keywords| keywords.children()) + .find(|pair| elixir_is_section_pair(pair, code, &["else:"])) + .and_then(|pair| pair.child_by_field_name("value")), + _ => None, + }; + section.is_some_and(|section| { + section.children().any(|clause| { + clause.kind_id() == E::StabClause as u16 + && !elixir_sole_unguarded_pattern(&clause) + .is_some_and(|pattern| elixir_is_bare_wildcard(&pattern, code)) + }) + }) + }) +} + +/// The keyword spellings of the clause sections a `do … end` block +/// spells as `do_block` / `else_block` / `after_block` / `rescue_block` +/// / `catch_block`. +const SECTION_KEYS: &[&str] = &["do:", "else:", "after:", "rescue:", "catch:"]; + +/// Whether `pair` is a keyword-form section (`do: (…)`) whose key is one +/// of `keys`. The key token's text carries its colon and the whitespace +/// after it, so it is trimmed before comparing. +fn elixir_is_section_pair(pair: &Node, code: &[u8], keys: &[&str]) -> bool { + pair.kind_id() == Elixir::Pair as u16 + && pair + .child_by_field_name("key") + .and_then(|key| key.utf8_text(code)) + .is_some_and(|key| keys.contains(&key.trim_end())) +} + +/// The `Call` whose `do` section holds `node` (a `stab_clause`): the +/// block form's `do_block` (whose parent is the call) or the keyword +/// form's `do: (…)` block (`block` → `pair` → `keywords` → `arguments` +/// → call). The caller asks the Call's keyword (`cond`, `for`). +fn elixir_do_section_call<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> Option> { + use Elixir as E; + + let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); + match up.next().map(|parent| parent.kind_id().into()) { + Some(E::DoBlock) => up.next(), + Some(E::Block) => up + .next() + .filter(|pair| elixir_is_section_pair(pair, code, &["do:"])) + .and_then(|_| up.nth(2)), + _ => None, + } +} diff --git a/src/metrics/elixir_named_operator_tests.rs b/src/metrics/elixir_named_operator_tests.rs new file mode 100644 index 00000000..1a8153f1 --- /dev/null +++ b/src/metrics/elixir_named_operator_tests.rs @@ -0,0 +1,144 @@ +//! A *named* Elixir operator scores like any other name (#1534). +//! +//! `&==/2`, `&and/2` and `Kernel.||(a, b)` name an operator: the grammar +//! wraps the token in an `operator_identifier`. Halstead billed both the +//! wrapper (an operand) and the token inside it (an operator), and +//! cyclomatic counted a captured `&&` / `||` / `and` / `or` as a decision. +//! Every row pairs the fixture with a twin naming an ordinary function, +//! which is the oracle; the twin's numbers are asserted outright too, so +//! a row cannot pass by both drifting. A remote call (`Kernel.||(a, b)`) +//! still applies the operator, so it keeps the decision; it is in the +//! applied table below. + +use std::cell::Cell; + +use crate::test_support::check_func_space_only; +use crate::*; + +/// `[n1, N1, n2, N2]`. +#[cfg(feature = "elixir")] +type Halstead = [u64; 4]; + +/// `[cyclomatic, cyclomatic_modified]` sums of the root space, which +/// carries a base of 1 each. +#[cfg(feature = "elixir")] +type Cyclomatic = [u64; 2]; + +#[cfg(feature = "elixir")] +fn measure(source: &str) -> (Halstead, Cyclomatic) { + let out = Cell::new(([0; 4], [0; 2])); + check_func_space_only::( + &format!("{source}\n"), + "foo.exs", + &[Metric::Halstead, Metric::Cyclomatic], + |space| { + let m = &space.metrics; + out.set(( + [ + m.halstead.unique_operators(), + m.halstead.total_operators(), + m.halstead.unique_operands(), + m.halstead.total_operands(), + ], + [ + m.cyclomatic.cyclomatic_sum(), + m.cyclomatic.cyclomatic_modified_sum(), + ], + )); + }, + ); + out.get() +} + +/// Whether `source` parses with an `operator_identifier`, so a row +/// whose fixture drifted to an ordinary name cannot pass by comparing +/// the twin with itself. +#[cfg(feature = "elixir")] +fn names_an_operator(source: &str) -> bool { + let parser = ElixirParser::new( + source.as_bytes().to_vec(), + std::path::Path::new("foo.exs"), + None, + ); + crate::test_support::ast_has_kind_id(&parser, Elixir::OperatorIdentifier as u16) +} + +/// Each named operator bills one operand — its `operator_identifier` — +/// and no operator, exactly as `foo` does in `&foo/2`, and decides +/// nothing. Before #1534 each row billed one operator more, and the +/// `&&` / `||` / `and` / `or` rows scored a decision in both tiers. +#[cfg(feature = "elixir")] +#[test] +fn elixir_named_operator_scores_like_a_named_function() { + // `(twin, named, halstead)`; every row decides nothing. + let rows: [(&str, &str, Halstead); 15] = [ + ("a = &foo/2", "a = &==/2", [3, 3, 3, 3]), + ("a = &foo/2", "a = &in/2", [3, 3, 3, 3]), + ("a = &foo/2", "a = ¬ in/2", [3, 3, 3, 3]), + ("a = &foo/2", "a = &and/2", [3, 3, 3, 3]), + ("a = &foo/2", "a = &or/2", [3, 3, 3, 3]), + ("a = &foo/2", "a = &||/2", [3, 3, 3, 3]), + ("a = &foo/1", "a = ¬/1", [3, 3, 3, 3]), + ("a = &foo/1", "a = &!/1", [3, 3, 3, 3]), + ("a = &foo/2", "a = &+/2", [3, 3, 3, 3]), + ("a = Kernel.foo(x, y)", "a = Kernel.==(x, y)", [4, 4, 5, 5]), + // A capture through the module names the operator too. + ("a = &Kernel.foo/2", "a = &Kernel.||/2", [4, 4, 4, 4]), + // `..` is a childless `operator_identifier`: the wrapper is the + // only node that can bill it, which is why it is the keeper. + ("a = foo", "a = ..", [1, 1, 2, 2]), + ("a = &foo/0", "a = &../0", [3, 3, 3, 3]), + // The lexer reads `&&&` as one token, so `&&&/2` holds no + // capture `&`: it is the name `&&&` divided by 2, like `foo/2`. + // `&&/1` lexes the same way, as the name `&&`. + ("a = foo/2", "a = &&&/2", [2, 2, 3, 3]), + ("a = foo/1", "a = &&/1", [2, 2, 3, 3]), + ]; + for (twin, named, halstead) in rows { + assert!( + names_an_operator(named), + "`{named}` lost its operator_identifier" + ); + let want = (halstead, [1, 1]); + assert_eq!(measure(twin), want, "`{twin}` moved; re-derive the row"); + assert_eq!(measure(named), want, "`{named}` must score like `{twin}`"); + } +} + +/// The operand is keyed by the operator's text, so two named operators +/// stay two names: `a`, `==`, `2`, `b`, `<`, with `2` twice. +#[cfg(feature = "elixir")] +#[test] +fn elixir_named_operators_are_distinct_operands() { + let (halstead, _) = measure("a = &==/2\nb = & Kernel.||(y)", [4, 4, 5, 5], [2, 2]), + ("a = Elixir.Kernel.||(x, y)", [4, 4, 5, 5], [2, 2]), + ("a = Foo.||(x, y)", [4, 4, 5, 5], [1, 1]), + ("a = x |> Foo.||(y)", [4, 4, 5, 5], [1, 1]), + ]; + for (source, halstead, cyclomatic) in rows { + assert_eq!(measure(source), (halstead, cyclomatic), "`{source}`"); + } +} diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index c0a49175..a54cfb58 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -75,7 +75,10 @@ pub struct HalsteadMaps<'a> { pub(crate) operators: IntKeyHashMap, /// Primitive-type operators stored by text so each distinct primitive /// (e.g. `int` vs `double`) counts as a separate distinct operator, - /// even when the grammar maps them all to a single kind_id. + /// even when the grammar maps them all to a single kind_id. Also + /// holds the operators a getter keys by a fixed spelling + /// ([`Getter::get_operator_spelling`]), such as the generic `<>` + /// pair (#1559). /// /// Text-keyed, so it keeps SipHash — see the module doc on /// [`crate::int_hash`] for why analysed source text does not qualify @@ -383,9 +386,25 @@ fn compute_halstead<'a, T: Getter + Checker>( ancestors: Ancestors<'a, '_>, halstead_maps: &mut HalsteadMaps<'a>, ) { + // Halstead counts the tokens written, and a zero-width node spells + // none, so it bills neither half (#1546). Error recovery is where + // they come from: every MISSING node tree-sitter inserts for a token + // the source lacks (a `;`, a `)`, an identifier) is zero-width, and + // so is the identifier C#'s and Elixir's recovery builds without + // flagging it MISSING. Checked here rather than per getter because + // every getter that classifies the kind would otherwise bill it, an + // identifier as the empty-string operand. + if node.start_byte() == node.end_byte() { + return; + } match T::get_op_type_with_code(node, code, ancestors) { TokenRole::Operator => { - if T::is_primitive(node) { + if let Some(spelling) = T::get_operator_spelling(node, ancestors) { + *halstead_maps + .primitive_operators + .entry(spelling.as_bytes()) + .or_insert(0) += 1; + } else if T::is_primitive(node) { // Store primitive-type operators by text so distinct // primitives (e.g. `int` vs `double`) that share a // single kind_id are counted separately in n1/N1. @@ -472,7 +491,7 @@ implement_metric_trait!(Halstead, PreprocCode, CcommentCode); clippy::too_many_lines )] mod tests { - use std::collections::HashSet; + use std::collections::{BTreeSet, HashSet}; use std::path::PathBuf; use crate::test_support::{ast_has_kind_id, check_metrics_only_shim, for_each_node_with_chain}; @@ -512,12 +531,15 @@ mod tests { feature = "groovy", feature = "irules", feature = "java", + feature = "javascript", feature = "kotlin", feature = "mozcpp", + feature = "mozjs", feature = "objc", feature = "perl", feature = "ruby", feature = "tcl", + feature = "typescript", ))] fn ops_of(source: &str, file: &str) -> crate::ops::Ops { let path = PathBuf::from(file); @@ -795,28 +817,31 @@ mod tests { c = 3 + 3", "foo.py", |metric| { - // unique operators: def, =, + - // operators: def, def, def, =, =, =, +, +, + + // unique operators: def, (), :, =, + + // operators: def, (), : for each of the three headers, + // then =, =, =, +, +, + (the `()` and `:` since #1486) // unique operands: foo, bar, toto, a, b, c, 1, 2, 3 // operands: foo, bar, toto, a, b, c, 1, 1, 2, a, 3, 3 + assert_eq!(metric.halstead.unique_operators(), 5); + assert_eq!(metric.halstead.total_operators(), 15); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 3, - "total_operators": 9, + "unique_operators": 5, + "total_operators": 15, "unique_operands": 9, "total_operands": 12, - "length": 21, - "estimated_program_length": 33.284212515144276, - "purity_ratio": 1.584962500721156, - "vocabulary": 12, - "volume": 75.28421251514428, - "difficulty": 2.0, - "level": 0.5, - "effort": 150.56842503028855, - "time": 8.364912501682698, - "bugs": 0.0094341190071077 + "length": 27, + "estimated_program_length": 40.13896548741762, + "purity_ratio": 1.4866283513858378, + "vocabulary": 14, + "volume": 102.79858289555531, + "difficulty": 3.3333333333333335, + "level": 0.3, + "effort": 342.6619429851844, + "time": 19.03677461028802, + "bugs": 0.01632259960095138 } "# ); @@ -1089,12 +1114,13 @@ mod tests { #[test] fn cpp_dot_star_is_halstead_operator() { check_metrics::("struct S { void operator.*(int); };", "foo.cpp", |metric| { - // Unique operators with fix: {}, ;, (), int, void, .* + // Unique operators with fix: {}, ;, (), int, void, .*, the + // `operator` keyword (#1296) and `struct` (#1552). // `.*` is the regression target — without the fix it - // falls through to `Unknown` and `u_operators` is 5. + // falls through to `Unknown` and `u_operators` is 7. // Unique operands: S let s = &metric.halstead; - assert_eq!(s.unique_operators(), 6); + assert_eq!(s.unique_operators(), 8); assert_eq!(s.unique_operands(), 1); }); } @@ -1116,12 +1142,13 @@ mod tests { "struct S { void operator->*(int); };", "foo.cpp", |metric| { - // Unique operators with fix: {}, ;, (), int, void, ->* + // Unique operators with fix: {}, ;, (), int, void, ->*, + // the `operator` keyword (#1296) and `struct` (#1552). // `->*` is the regression target — without the fix it - // falls through to `Unknown` and `u_operators` is 5. + // falls through to `Unknown` and `u_operators` is 7. // Unique operands: S let s = &metric.halstead; - assert_eq!(s.unique_operators(), 6); + assert_eq!(s.unique_operators(), 8); assert_eq!(s.unique_operands(), 1); }, ); @@ -1256,9 +1283,14 @@ mod tests { // operand counts pin the rest of the Halstead state. // Grew from 30 → 33 with the issue #394 fix: `const`, // `type`, and `struct` keywords are now classified as - // operators (one occurrence each). + // operators (one occurrence each). #1486 adds the + // annotation `:` — one entry, 23 occurrences: `C`, the + // two fields, `p`, the four params of `f`, and the + // fifteen `let` bindings. #1559 bills `Option`'s + // brackets as one `<>` pair: one entry and one + // occurrence fewer. assert_eq!(metric.halstead.unique_operators(), 33); - assert_eq!(metric.halstead.total_operators(), 121); + assert_eq!(metric.halstead.total_operators(), 143); // u_operands / operands grew (was 31/50 before #390): the // fix now classifies TypeIdentifier (`T`, `S`, `Option`) // and FieldIdentifier (struct fields `x`, `y`) as operands @@ -1277,7 +1309,7 @@ mod tests { // `TokenRole::Unknown`, so the field names were not counted // as operands. Both C++ and Go already classify FieldIdentifier // as an operand. After the fix: - // unique operators: fn, (), {}, let, =, +, ;, . + // unique operators: fn, (), {}, let, =, +, ;, ., ,, : // unique operands : main, p, Point, x, y, sum, 0, 1 // Field names `x` and `y` each appear twice (`p.x + p.y` and // the struct literal `Point { x: 0, y: 1 }`). @@ -1294,24 +1326,28 @@ mod tests { // fix, +Point, +x, +y → 8 distinct names. assert_eq!(metric.halstead.unique_operands(), 8); assert_eq!(metric.halstead.total_operands(), 12); + // N1: fn, (), {} ×2, let ×2, = ×2, +, ; ×2, . ×2, `,`, + // and the struct literal's two field `:` (#1486) = 16. + assert_eq!(metric.halstead.unique_operators(), 10); + assert_eq!(metric.halstead.total_operators(), 16); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 9, - "total_operators": 14, + "unique_operators": 10, + "total_operators": 16, "unique_operands": 8, "total_operands": 12, - "length": 26, - "estimated_program_length": 52.529325012980806, - "purity_ratio": 2.0203586543454155, - "vocabulary": 17, - "volume": 106.27403387250882, - "difficulty": 6.75, - "level": 0.14814814814814814, - "effort": 717.3497286394346, - "time": 39.85276270219081, - "bugs": 0.026711567292222575 + "length": 28, + "estimated_program_length": 57.219280948873624, + "purity_ratio": 2.043545748174058, + "vocabulary": 18, + "volume": 116.75790004038474, + "difficulty": 7.5, + "level": 0.13333333333333333, + "effort": 875.6842503028855, + "time": 48.64912501682697, + "bugs": 0.03051010983631521 } "# ); @@ -1331,7 +1367,10 @@ mod tests { // // Also covers issue #394: `::` is now an operator. The snippet // has two `::` tokens (`Vec::new`, `HashMap::new`), so n1 grew - // from 10 → 11 and N1 from 17 → 19. + // from 10 → 11 and N1 from 17 → 19. #1486 then billed the two + // annotation `:` tokens as their own entry: 12 and 21. #1559 + // billed each generic's brackets as one `<>` pair, replacing the + // `<` and `>` entries and their four occurrences: 11 and 19. check_metrics::( "fn main() { let v: Vec = Vec::new(); @@ -1346,6 +1385,7 @@ mod tests { assert_eq!(metric.halstead.total_operands(), 11); // `::` appears twice (Vec::new, HashMap::new); without // the #394 fix u_operators was 10 and operators 17. + // `:` appears twice and stays distinct from `::`. assert_eq!(metric.halstead.unique_operators(), 11); assert_eq!(metric.halstead.total_operators(), 19); insta::assert_json_snapshot!( @@ -1373,6 +1413,45 @@ mod tests { ); } + /// #1486: Rust's single `:` is vocabulary in every position it is + /// spelled, and stays a separate entry from `::`. + #[cfg(feature = "rust")] + #[test] + fn rust_colon_is_operator_1486() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `let`, `:`, `u8`, `=`, `;` + ("let annotation", "let x: u8 = 1;\n", [5, 5]), + // `struct`, `{}`, `:`, `u8` + ("field declaration", "struct S { a: u8 }\n", [4, 4]), + // `const`, `:` twice, `=`, `{}`, `;` + ("field initializer", "const C: S = S { a: 1 };\n", [5, 6]), + // `fn`, `{}` twice, `()`, `let`, `{}` (pattern), `:`, `=`, `;` + ("field pattern", "fn f() { let S { a: b } = s; }\n", [7, 8]), + // `fn`, `()`, `:`, `u8`, `{}` + ("parameter", "fn f(x: u8) {}\n", [5, 5]), + // `fn`, `<>`, `:`, `()`, `{}` + ("trait bound", "fn f() {}\n", [5, 5]), + // `fn`, `<>`, `()`, `where`, `:`, `{}` + ("where clause", "fn f() where T: Clone {}\n", [6, 6]), + // `fn`, `()`, `{}` twice, `:`, `loop`, `break`, `;` + ("loop label", "fn f() { 'a: loop { break 'a; } }\n", [7, 8]), + // `fn`, `()`, `{}`, `let`, `=`, `|` twice, `:`, `u8`, `;` + ( + "closure parameter", + "fn f() { let c = |q: u8| q; }\n", + [9, 10], + ), + // `fn`, `()` twice, `{}`, `let`, `:`, `<>`, `u8`, `=`, `::`, + // `;` — `:` and `::` are two entries + ( + "beside a path", + "fn f() { let v: Vec = Vec::new(); }\n", + [10, 11], + ), + ]; + assert_operator_rows::("foo.rs", ROWS); + } + #[cfg(feature = "rust")] #[test] fn rust_path_separator_is_operator() { @@ -1714,6 +1793,139 @@ mod tests { ); } + /// One JSX shape per row of #1483's decision: the source, the + /// expected `n2` / `N2`, and the deduplicated operand vocabulary. + /// Every row is a single element bound to `a`, so the operands + /// outside the element's content are always `a` plus the tag names. + #[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))] + const JSX_TEXT_CASES: &[(&str, &str, [u64; 2], &[&str])] = &[ + // Element text is one operand. N2: a, p x2 (both tags), hi. + ("text", "const a =

hi

;\n", [3, 4], &["a", "p", "hi"]), + // The same-line space between `` and `` is a jsx_text + // node of its own, and bills nothing. Billing it would key an + // empty operand: (5, 6). + ( + "whitespace", + "const a =

;\n", + [4, 5], + &["a", "p", "b", "i"], + ), + // `Start` at two indentation depths is one distinct operand, + // because the key is the trimmed text. Untrimmed: (5, 7). + ( + "indentation", + "const a = (\n

\n Start\n \n Start\n \n

\n);\n", + [4, 7], + &["a", "p", "span", "Start"], + ), + // An entity splits the text into three siblings, each an + // operand. Without the entity arm: (4, 5). + ( + "entity in text", + "const a =

x & y

;\n", + [5, 6], + &["a", "p", "x", "&", "y"], + ), + // Inside an attribute string the entity is part of the string + // operand and bills nothing of its own. Without the parent + // guard: (5, 5). + ( + "entity in attribute", + "const a =

;\n", + [4, 4], + &["a", "p", "title", "\"x & y\""], + ), + // An expression child is unchanged by #1483: its identifier was + // already an operand. + ( + "expression", + "const a =

{v}

;\n", + [3, 4], + &["a", "p", "v"], + ), + ]; + + /// Asserts every row of [`JSX_TEXT_CASES`] under parser `T`, through + /// the metrics store (`n2` / `N2`) and the `--ops` vocabulary. + #[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))] + #[track_caller] + fn assert_jsx_text_operands(file: &str) { + for (label, source, expected, vocabulary) in JSX_TEXT_CASES { + crate::test_support::check_func_space_only::( + source, + file, + &[crate::Metric::Halstead], + |space| { + let halstead = &space.metrics.halstead; + assert_eq!( + [halstead.unique_operands(), halstead.total_operands()], + *expected, + "{file}: {label}" + ); + }, + ); + let ops = ops_of::(source, file); + let got: BTreeSet<&str> = ops.operands.iter().map(String::as_str).collect(); + let want: BTreeSet<&str> = vocabulary.iter().copied().collect(); + assert_eq!(got, want, "{file}: {label} operand vocabulary"); + } + } + + /// Regression for #1483: JSX element text and its character + /// references are operands. One row per grammar, so reverting one + /// invocation's `jsx:` argument fails its own row. + #[cfg(all(feature = "javascript", feature = "mozjs", feature = "typescript"))] + #[test] + fn jsx_text_is_an_operand() { + // A whitespace-only node bills nothing, so no count can tell the + // "whitespace" row's fixture from one that stopped spelling the + // same-line space. Pin the node's presence directly. + let (_, source, _, _) = JSX_TEXT_CASES + .iter() + .find(|(label, ..)| *label == "whitespace") + .expect("the whitespace row exists"); + let parser = + JavascriptParser::new(source.as_bytes().to_vec(), &PathBuf::from("foo.jsx"), None); + assert!( + parser.root().preorder().any(|n| { + n.kind_id() == Javascript::JsxText as u16 + && source.as_bytes()[n.start_byte()..n.end_byte()] + .trim_ascii() + .is_empty() + }), + "the whitespace row must carry a whitespace-only jsx_text node" + ); + + assert_jsx_text_operands::("foo.jsx"); + assert_jsx_text_operands::("foo.js"); + assert_jsx_text_operands::("foo.tsx"); + } + + /// The `.ts` grammar's enum carries `JsxText` in the externals it + /// shares with TSX, but no production emits it, which is why the + /// TypeScript invocation passes no `jsx:` argument (#1483). Pinned + /// so a grammar that starts emitting it cannot go unbilled quietly. + #[cfg(feature = "typescript")] + #[test] + fn typescript_never_emits_jsx_text() { + let parser = TypescriptParser::new( + b"const a =

hi

;\nlet b = (x);\n".to_vec(), + &PathBuf::from("foo.ts"), + None, + ); + // The positive half: the parse reached the angle-bracket syntax + // TSX would read as JSX, so the absence below is a property of + // the grammar rather than of a parse that produced nothing. + assert!( + ast_has_kind_id(&parser, Typescript::TypeAssertion as u16), + "fixture lost the `(x)` type assertion" + ); + assert!( + !ast_has_kind_id(&parser, Typescript::JsxText as u16), + "the .ts grammar now emits jsx_text; give its invocation a `jsx:` argument" + ); + } + #[cfg(feature = "typescript")] #[test] fn typescript_template_string_plain_is_operand() { @@ -2096,37 +2308,29 @@ mod tests { // // expected, for `class C { #x = 1; m() { return this.#x; } }`: // - // * Operators: `{`×2 (class body, method body), `=`, `;`×2, `(`, - // `return`, `.` — 8 total, 6 unique. (`class` is not in the - // JS-family operator arm, so it contributes nothing; that is - // pre-existing and unrelated.) + // * Operators: `class`, `{`×2 (class body, method body), `=`, + // `;`×2, `(`, `return`, `.` — 9 total, 7 unique. (`class` joined + // the JS-family operator arm in #1552.) // * Operands: `C`, `#x`, `1`, `m`, `this`, `#x` — 6 total, 5 - // unique under JS/MozJS. Under TS/TSX the class *name* `C` - // parses as `type_identifier`, which those getters do not - // classify, so both counts drop by one to 5/4 — a pre-existing - // divergence this fixture records rather than fixes. + // unique in all four grammars. Under TS/TSX the class *name* `C` + // is a `type_identifier`, which those getters did not classify + // until #1557, so they read 5/4 before it. #[cfg(all(feature = "javascript", feature = "mozjs", feature = "typescript"))] #[test] fn js_family_private_field_leaf_is_the_operand_1263() { #[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))] const SRC: &str = "class C { #x = 1; m() { return this.#x; } }"; - let check_js = |m: crate::CodeMetrics| { - assert_eq!(m.halstead.unique_operators(), 6); - assert_eq!(m.halstead.total_operators(), 8); + let check = |m: crate::CodeMetrics| { + assert_eq!(m.halstead.unique_operators(), 7); + assert_eq!(m.halstead.total_operators(), 9); assert_eq!(m.halstead.unique_operands(), 5); assert_eq!(m.halstead.total_operands(), 6); }; - let check_ts = |m: crate::CodeMetrics| { - assert_eq!(m.halstead.unique_operators(), 6); - assert_eq!(m.halstead.total_operators(), 8); - assert_eq!(m.halstead.unique_operands(), 4); - assert_eq!(m.halstead.total_operands(), 5); - }; - check_metrics::(SRC, "foo.js", check_js); - check_metrics::(SRC, "foo.js", check_js); - check_metrics::(SRC, "foo.ts", check_ts); - check_metrics::(SRC, "foo.tsx", check_ts); + check_metrics::(SRC, "foo.js", check); + check_metrics::(SRC, "foo.js", check); + check_metrics::(SRC, "foo.ts", check); + check_metrics::(SRC, "foo.tsx", check); } // Issue #1263, the other section 6 half: `meta_property` is the one @@ -2163,8 +2367,8 @@ mod tests { // // expected, for `namespace N.M { }`: // - // * Operators: `.`, `{` — 2 total, 2 unique. (`namespace` is not in - // the JS-family operator arm.) + // * Operators: `namespace`, `.`, `{` — 3 total, 3 unique. + // (`namespace` joined the TS operator extras in #1557.) // * Operands: `N`, `M` — 2 total, 2 unique. Before the fix the // `nested_identifier` added `N.M`, making both 3. #[cfg(feature = "typescript")] @@ -2172,8 +2376,8 @@ mod tests { fn ts_nested_identifier_counts_leaves_not_the_composite_1263() { const SRC: &str = "namespace N.M { }"; let check = |m: crate::CodeMetrics| { - assert_eq!(m.halstead.unique_operators(), 2); - assert_eq!(m.halstead.total_operators(), 2); + assert_eq!(m.halstead.unique_operators(), 3); + assert_eq!(m.halstead.total_operators(), 3); assert_eq!(m.halstead.unique_operands(), 2); assert_eq!(m.halstead.total_operands(), 2); }; @@ -2397,20 +2601,26 @@ mod tests { #[cfg(feature = "python")] #[test] fn python_wrong_operators() { + // Not valid Python — the parser recovers `()` as a tuple, the `[` + // as a bare token and `{}` as a dict — but each opener is still a + // delimiter the source spells, so since #1486 the three are + // billed once each as `()`, `[]` and `{}`. The closers never are. check_metrics::("()[]{}", "foo.py", |metric| { + assert_eq!(metric.halstead.unique_operators(), 3); + assert_eq!(metric.halstead.total_operators(), 3); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 0, - "total_operators": 0, + "unique_operators": 3, + "total_operators": 3, "unique_operands": 0, "total_operands": 0, - "length": 0, - "estimated_program_length": 0.0, - "purity_ratio": 0.0, - "vocabulary": 0, - "volume": 0.0, + "length": 3, + "estimated_program_length": 4.754887502163468, + "purity_ratio": 1.584962500721156, + "vocabulary": 3, + "volume": 4.754887502163468, "difficulty": 0.0, "level": 0.0, "effort": 0.0, @@ -2430,24 +2640,28 @@ mod tests { pass", "foo.py", |metric| { + // expected operators: def, (), :, pass — one each (#1486 + // added the `()` and the block `:`). + assert_eq!(metric.halstead.unique_operators(), 4); + assert_eq!(metric.halstead.total_operators(), 4); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 2, - "total_operators": 2, + "unique_operators": 4, + "total_operators": 4, "unique_operands": 1, "total_operands": 1, - "length": 3, - "estimated_program_length": 2.0, - "purity_ratio": 0.6666666666666666, - "vocabulary": 3, - "volume": 4.754887502163468, - "difficulty": 1.0, - "level": 1.0, - "effort": 4.754887502163468, - "time": 0.26416041678685936, - "bugs": 0.0009425525573729414 + "length": 5, + "estimated_program_length": 8.0, + "purity_ratio": 1.6, + "vocabulary": 5, + "volume": 11.60964047443681, + "difficulty": 2.0, + "level": 0.5, + "effort": 23.21928094887362, + "time": 1.289960052715201, + "bugs": 0.002712967490108627 } "# ); @@ -2469,26 +2683,27 @@ mod tests { }", "foo.java", |metric| { - // Operators (n1=11): {} void () [] , . ; int = + / - // Operands (n2=12): Main main args a b c avg 5 3 MessageFormat format "{0}" + // Operators (n1=12): class {} void () [] , . ; int = + / + // Operands (n2=13): Main main string args a b c avg 5 3 MessageFormat format "{0}" + // (`string` is the parameter's type name, an operand since #1560) insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 11, - "total_operators": 26, - "unique_operands": 12, - "total_operands": 22, - "length": 48, - "estimated_program_length": 81.07329781366414, - "purity_ratio": 1.6890270377846697, - "vocabulary": 23, - "volume": 217.13097389073664, - "difficulty": 10.083333333333334, - "level": 0.09917355371900825, - "effort": 2189.4039867315946, - "time": 121.63355481842193, - "bugs": 0.05620341201461669 + "unique_operators": 12, + "total_operators": 27, + "unique_operands": 13, + "total_operands": 23, + "length": 50, + "estimated_program_length": 91.12526634448807, + "purity_ratio": 1.8225053268897613, + "vocabulary": 25, + "volume": 232.19280948873623, + "difficulty": 10.615384615384615, + "level": 0.09420289855072464, + "effort": 2464.8159776496614, + "time": 136.93422098053674, + "bugs": 0.06082308053869885 } "# ); @@ -2515,25 +2730,26 @@ mod tests { |metric| { // Verifies all 8 Java primitive-type keywords (byte, short, int, long, // char, float, double, boolean) are counted as distinct operators, and - // that true/false are counted as operands. + // that true/false are counted as operands. The other four operators + // are `class` (#1552), `{}`, `=` and `;`. insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 11, - "total_operators": 28, + "unique_operators": 12, + "total_operators": 29, "unique_operands": 19, "total_operands": 19, - "length": 47, - "estimated_program_length": 118.76437056043838, - "purity_ratio": 2.526901501285923, - "vocabulary": 30, - "volume": 230.62385799360038, - "difficulty": 5.5, - "level": 0.18181818181818182, - "effort": 1268.4312189648022, - "time": 70.46840105360012, - "bugs": 0.03905920146699976 + "length": 48, + "estimated_program_length": 123.730172764082, + "purity_ratio": 2.5777119325850415, + "vocabulary": 31, + "volume": 237.80142289857, + "difficulty": 6.0, + "level": 0.16666666666666666, + "effort": 1426.80853739142, + "time": 79.26714096619, + "bugs": 0.04224635182363915 } "# ); @@ -2552,14 +2768,15 @@ mod tests { #[test] fn java_self_and_super_references_are_operands() { let source = "class T {\n int f() { return this.x + super.y; }\n}"; - // Operators (n1=7, N1=9): {} x2, int, (), return, . x2, +, ; + // Operators (n1=8, N1=10): class (#1552), {} x2, int, (), + // return, . x2, +, ; // Operands (n2=6, N2=6): T, f, this, x, super, y - // Before the fix this read (9, 11, 4, 4) — `this` and `super` + // Before the fix this read (10, 12, 4, 4) — `this` and `super` // billed into the operator side of both counts. assert_halstead_counts::( source, "foo.java", - [7, 9, 6, 6], + [8, 10, 6, 6], "java self/super references", ); assert_keywords_are_operands_only::(source, "foo.java", &["this", "super"]); @@ -2734,26 +2951,32 @@ mod tests { // not already counted as operands, since `String` // was already an identifier in the prior grammar's // counting). - assert_eq!(metric.halstead.unique_operators(), 8); + // `class` is the ninth unique operator since #1552. + assert_eq!(metric.halstead.unique_operators(), 9); assert_eq!(metric.halstead.unique_operands(), 13); + // `bugs` is `effort.powf(2/3) / 3000`, and the platform libm's + // `powf` differs in the last ulp: macOS rendered this value + // as `…92342184` where Linux and Windows gave `…923421845`. + // Round it so the snapshot pins the metric, not the libm. insta::assert_json_snapshot!( metric.halstead, + { ".bugs" => insta::rounded_redaction(12) }, @r#" { - "unique_operators": 8, - "total_operators": 22, + "unique_operators": 9, + "total_operators": 23, "unique_operands": 13, "total_operands": 23, - "length": 45, - "estimated_program_length": 72.10571633583419, - "purity_ratio": 1.6023492519074265, - "vocabulary": 21, - "volume": 197.65428402504423, - "difficulty": 7.076923076923077, - "level": 0.14130434782608697, - "effort": 1398.7841638695438, - "time": 77.71023132608576, - "bugs": 0.04169134280255714 + "length": 46, + "estimated_program_length": 76.63504134881501, + "purity_ratio": 1.665979159756848, + "vocabulary": 22, + "volume": 205.13385445731566, + "difficulty": 7.961538461538462, + "level": 0.12560386473429952, + "effort": 1633.1810720255517, + "time": 90.73228177919731, + "bugs": 0.046227630923 } "# ); @@ -2787,29 +3010,29 @@ mod tests { // split), and `'x'` parses as `StringLiteral` (Groovy // single-quoted strings) rather than as // `CharacterLiteral`. Operators remaining in this - // fixture: `=` and `class`-body braces (only `{` is in - // the operator set). True/false collapse under one - // `BooleanLiteral`. - assert_eq!(metric.halstead.unique_operators(), 2); + // fixture: `=`, the `class` keyword (#1552) and the + // class-body braces (only `{` is in the operator set). + // True/false collapse under one `BooleanLiteral`. + assert_eq!(metric.halstead.unique_operators(), 3); assert_eq!(metric.halstead.unique_operands(), 27); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 2, - "total_operators": 10, + "unique_operators": 3, + "total_operators": 11, "unique_operands": 27, "total_operands": 28, - "length": 38, - "estimated_program_length": 130.38196255841365, - "purity_ratio": 3.4311042778529908, - "vocabulary": 29, - "volume": 184.60327781484773, - "difficulty": 1.037037037037037, - "level": 0.9642857142857143, - "effort": 191.44043625243467, - "time": 10.635579791801925, - "bugs": 0.01107221547116606 + "length": 39, + "estimated_program_length": 133.1368500605771, + "purity_ratio": 3.413765386168644, + "vocabulary": 30, + "volume": 191.36873322873222, + "difficulty": 1.5555555555555556, + "level": 0.6428571428571428, + "effort": 297.6846961335835, + "time": 16.53803867408797, + "bugs": 0.014861058740943886 } "# ); @@ -3020,6 +3243,37 @@ mod tests { assert_ops_operands::(src, "foo.groovy", 3, vec!["greet", "name", "$name"]); } + // `Groovy::Identifier2` (141) is the grammar's external token + // `_gstring_dollar_path` — the `$a.b` path of a dollar interpolation — + // whose public symbol is `identifier`, so the parser reports it as + // the unsuffixed `Identifier` and never as 141 (#1475). That is why + // the operand arm lists `Identifier` but not `Identifier2`; if a + // grammar bump starts emitting 141, the dotted path drops out of N2 + // with no error. + // + // expected: operands `greet`, `name` (param), `$name.b` (the whole + // path, one node) → n2 = 3, N2 = 3. + #[cfg(feature = "groovy")] + #[test] + fn groovy_gstring_dollar_path_is_the_public_identifier() { + let src = "def greet(name) {\n return \"Hi $name.b\"\n}\n"; + let parser = GroovyParser::new(src.as_bytes().to_vec(), &PathBuf::from("foo.groovy"), None); + assert!( + ast_has_kind_id(&parser, Groovy::GstringDollarInterpolation as u16), + "the fixture no longer spells a dollar interpolation", + ); + assert!( + !ast_has_kind_id(&parser, Groovy::Identifier2 as u16), + "the hidden `_gstring_dollar_path` token is now emitted as \ + `Identifier2`; add it to the Groovy operand arm", + ); + check_metrics::(src, "foo.groovy", |metric| { + assert_eq!(metric.halstead.unique_operands(), 3); + assert_eq!(metric.halstead.total_operands(), 3); + }); + assert_ops_operands::(src, "foo.groovy", 3, vec!["greet", "name", "$name.b"]); + } + #[cfg(feature = "groovy")] #[test] fn groovy_plain_string_still_operand() { @@ -3201,17 +3455,17 @@ mod tests { assert_eq!(metric.halstead.total_operands(), 2); }); - // expected: operators `class`, `{`×2, `void`, `(`, `;`, `<`, - // `>`, `int` — 9 total, 8 unique (`int` is the text-keyed - // primitive operator, per #286). Operands `C`, `M`, `List`, `l` + // expected: operators `class`, `{`×2, `void`, `(`, `;`, `<>`, + // `int` — 8 total, 7 unique (`int` is the text-keyed primitive + // operator, per #286; `<>` is the type-argument pair, #1559). Operands `C`, `M`, `List`, `l` // — 4/4. Pre-fix the `generic_name` added `List`, making // them 5/5. check_metrics::( "class C { void M() { List l; } }", "foo.cs", |metric| { - assert_eq!(metric.halstead.unique_operators(), 8); - assert_eq!(metric.halstead.total_operators(), 9); + assert_eq!(metric.halstead.unique_operators(), 7); + assert_eq!(metric.halstead.total_operators(), 8); assert_eq!(metric.halstead.unique_operands(), 4); assert_eq!(metric.halstead.total_operands(), 4); }, @@ -3308,33 +3562,45 @@ mod tests { #[cfg(feature = "csharp")] #[test] - fn csharp_boolean_keyword_outside_a_literal_still_counts() { - // Companion to the test above (#1253): the suppression fires on - // the *parent* kind, never on `True` / `False` alone. C#'s - // overloadable-operator list emits a bare `true` / `false` token - // with no `boolean_literal` wrapper — `operator_declaration` is - // the grammar's only such position — so a blanket exclusion - // would drop the operand that is the sole difference between - // `operator true` and `operator false`, leaving two such - // declarations with identical Halstead vocabularies whenever - // their bodies match. + fn csharp_operator_true_names_an_operator_not_a_value() { + // C#'s overloadable-operator list emits a bare `true` / `false` + // token with no `boolean_literal` wrapper — `operator_declaration` + // is the grammar's only such position. There the keyword names + // the operator being declared, as `+` does in `operator +`, so it + // is an operator (#1296); the companion test above pins the + // wrapped literal, which bills only its wrapper (#1253). // // Each declaration names one boolean and returns the other, so - // the fixture exercises both the guarded and the unguarded - // position for each keyword. - // - // Operands: `A` × 3 (class name, two parameter types), `a` × 2, - // `true` × 2 (operator name + literal), `false` × 2 (likewise) - // ⇒ n2 = 4, N2 = 9. A blanket exclusion gives N2 = 7; no guard - // at all restores the double count at N2 = 11. - check_metrics::( - "class A {\n public static bool operator true(A a) => false;\n public static bool operator false(A a) => true;\n}\n", - "foo.cs", - |metric| { - assert_eq!(metric.halstead.unique_operands(), 4); - assert_eq!(metric.halstead.total_operands(), 9); - }, - ); + // the fixture carries each keyword once as a name and once as a + // value. + // + // Operators (n1 = 11, N1 = 18): class 1, {} 1, public 2, + // static 2, bool 2, operator 2, true 1, false 1, () 2, => 2, + // ; 2. Operands (n2 = 4, N2 = 7): A × 3 (class name, two + // parameter types), a × 2, and the literals false and true once + // each. Before #1296 the two names were operands instead: + // (9, 16, 4, 9). A blanket exclusion of the leaf would give + // (9, 16, 4, 7), and no guard at all (11, 18, 4, 9). + let source = "class A {\n public static bool operator true(A a) => false;\n public static bool operator false(A a) => true;\n}\n"; + assert_halstead_counts::(source, "foo.cs", [11, 18, 4, 7], "operator true"); + + // `true` and `false` are each an operator *and* an operand here, + // once apiece: the name and the literal land in different halves. + let ops = ops_of::(source, "foo.cs"); + for keyword in ["true", "false"] { + for (half, list) in [("operators", &ops.operators), ("operands", &ops.operands)] { + assert_eq!( + list.iter().filter(|o| *o == keyword).count(), + 1, + "`{keyword}` must appear once among the {half}; they were {list:?}" + ); + } + } + + // The fallback arm: outside both positions — here a bare `true` + // under error recovery, the only other parent the grammar + // produces — the keyword keeps its role as a value. + assert_halstead_counts::("true\n", "foo.cs", [0, 0, 1, 1], "bare true"); } #[cfg(feature = "csharp")] @@ -3647,48 +3913,63 @@ mod tests { // assertion is dead. That left four of the five bare kinds with // no input at all — dropping `Csharp::Ref`, `In`, `Readonly` or // `Scoped` from `is_primitive` failed no test in the workspace. - let both_code = CSHARP_MODIFIER_BOTH_SPELLINGS.as_bytes(); - let spellings = [ - ("ref", Csharp::Ref as u16), - ("out", Csharp::Out as u16), - ("in", Csharp::In as u16), - ("scoped", Csharp::Scoped as u16), - ("readonly", Csharp::Readonly as u16), - ]; - let (mut bare, mut aliased) = ([0_usize; 5], [0_usize; 5]); + assert_modifier_spellings_are_one_operator( + CSHARP_MODIFIER_BOTH_SPELLINGS, + &[ + ("ref", Csharp::Ref as u16), + ("out", Csharp::Out as u16), + ("in", Csharp::In as u16), + ("scoped", Csharp::Scoped as u16), + ("readonly", Csharp::Readonly as u16), + ], + ); + } - for_each_node_with_chain::(both_code, |node, _| { - let text = &both_code[node.start_byte()..node.end_byte()]; + /// Asserts each keyword in `spellings` appears in `source` both as + /// its own `kind` and as a childless aliased `modifier`, and that the + /// two spellings render as exactly one entry in the operator + /// vocabulary. + /// + /// The vocabulary is the concatenation of the kind-keyed and the + /// lexeme-keyed operator maps, so a keyword listed twice is one + /// keyword split across both maps and counted twice in `n1`. + #[cfg(feature = "csharp")] + #[track_caller] + fn assert_modifier_spellings_are_one_operator(source: &str, spellings: &[(&str, u16)]) { + let code = source.as_bytes(); + // (bare, aliased) occurrences per keyword. + let mut seen = vec![(0_usize, 0_usize); spellings.len()]; + + for_each_node_with_chain::(code, |node, _| { + let text = &code[node.start_byte()..node.end_byte()]; let Some(index) = spellings.iter().position(|(kw, _)| kw.as_bytes() == text) else { return; }; if node.kind_id() == Csharp::Modifier as u16 && node.child_count() == 0 { - aliased[index] += 1; + seen[index].1 += 1; } else if node.kind_id() == spellings[index].1 { - bare[index] += 1; + seen[index].0 += 1; } }); - // Without this the count below is vacuous for any keyword the - // fixture stopped spelling both ways — the same decay that hid - // the four unguarded kinds in the first place. - for (index, (keyword, _)) in spellings.iter().enumerate() { + // Without this the vocabulary count below is vacuous for any + // keyword the fixture stopped spelling both ways — the decay + // that once left four of #1418's bare kinds with no input. + for ((keyword, _), (bare, aliased)) in spellings.iter().zip(&seen) { assert!( - bare[index] > 0 && aliased[index] > 0, - "fixture must spell `{keyword}` both ways; it has {} bare and {} aliased", - bare[index], - aliased[index] + *bare > 0 && *aliased > 0, + "fixture must spell `{keyword}` both ways; it has {bare} bare and {aliased} aliased" ); } - let both = ops_of::(CSHARP_MODIFIER_BOTH_SPELLINGS, "both.cs"); + let ops = ops_of::(source, "both.cs"); for (keyword, _) in spellings { assert_eq!( - both.operators.iter().filter(|o| *o == keyword).count(), + ops.operators.iter().filter(|o| o == keyword).count(), 1, "`{keyword}` must be one operator across both of its kind spellings; \ operators were {:?}", - both.operators + ops.operators ); } } @@ -3707,11 +3988,8 @@ mod tests { /// Childlessness says "aliased", *not* "parameter modifier" — /// `_lambda_expression_init` and `anonymous_method_expression` alias /// a bare `static` / `async` onto the same kind, which the second - /// walk below pins so the distinction cannot be forgotten. Their - /// `Unknown` is today's behaviour rather than a contract: it is what - /// they were before #1418, and billing them is a decision about - /// lambdas. It is asserted only so making that decision has to come - /// through this test. + /// walk below pins so the distinction cannot be forgotten. Both are + /// operators, as their declaration spelling is (#1482). #[cfg(feature = "csharp")] #[test] fn csharp_aliased_modifier_is_childless() { @@ -3728,9 +4006,8 @@ mod tests { if node.child_count() == 0 { aliased += 1; // Spelled out rather than `this`-versus-everything-else, - // because "everything else is an operator" is false of - // the kind at large — a lambda's aliased `static` is - // neither, and the second walk below owns it. + // so a text this fixture does not carry fails loudly; + // the lambda spellings are owned by the second walk. let (correct, expected) = match text { b"this" => (matches!(role, TokenRole::Operand), "an operand"), b"scoped" | b"ref" | b"out" | b"in" | b"readonly" => { @@ -3830,10 +4107,9 @@ mod tests { assert!( matches!( CsharpCode::get_op_type_with_code(node, lambda, Ancestors::known(chain)), - TokenRole::Unknown + TokenRole::Operator ), - "`{}` on a lambda is unbilled today, as it was before #1418; billing it is a \ - deliberate change and belongs in this assertion, not around it", + "`{}` on a lambda must be an operator, as its declaration spelling is (#1482)", String::from_utf8_lossy(text) ); }); @@ -3844,6 +4120,88 @@ mod tests { ); } + /// `static` and `async` in both of their kind spellings: the keyword + /// leaf under a method's `modifier` wrapper, and the childless + /// `modifier` the grammar aliases a lambda's keyword to (#1482). + #[cfg(feature = "csharp")] + const CSHARP_LAMBDA_MODIFIER_BOTH_SPELLINGS: &str = "class L { + static void N() { } + async void M() { } + void P() { + var a = static (int x) => x; + var b = async (int y) => y; + } +}"; + + /// A C# lambda's bare parameter (`x => x`) is the operand its + /// parenthesised twin (`(x) => x`) bills (#1544). Each row is one + /// lambda in a method body; the twins differ only by the `()` the + /// parenthesised spelling writes, so the operand halves must match + /// and the operator halves differ by exactly that one `()`. + #[cfg(feature = "csharp")] + #[test] + fn csharp_implicit_lambda_parameter_is_an_operand() { + let wrap = |lambda: &str| { + format!("class C {{\n void M() {{\n var f = {lambda};\n }}\n}}\n") + }; + // Operators: class, {} x2, void, () x1 (M's parameter list), =, + // =>, ; — n1 = 7, N1 = 8 — plus the lambda's own `()` in the + // parenthesised spelling. Operands: C, M, f, and the parameter + // twice; `var` is an unclassified `implicit_type`. `_ => 0` + // names `_` once and adds the literal `0`. + // + // Before #1544 the bare rows read (7, 8, 4, 4) for both: `x` + // lost one occurrence, and `_` left the vocabulary entirely. + for (lambda, expected) in [ + ("x => x", [7, 8, 4, 5]), + ("(x) => x", [7, 9, 4, 5]), + ("_ => 0", [7, 8, 5, 5]), + ("(_) => 0", [7, 9, 5, 5]), + ] { + assert_halstead_counts::(&wrap(lambda), "foo.cs", expected, lambda); + } + + // The bare parameter is billed under its own text, not under + // the kind name, and as an operand only. + assert_keywords_are_operands_only::(&wrap("_ => 0"), "foo.cs", &["_"]); + } + + /// A lambda's `static` / `async` is the same operator as the + /// declaration's, counted once in `n1` across both spellings + /// (#1482). + #[cfg(feature = "csharp")] + #[test] + fn csharp_lambda_modifiers_are_operators_1482() { + // Operators (n1 = 10, N1 = 25): class 1; {} 4 (the class body + // and three method bodies); static 2; async 2; void 3; () 5 + // (three method parameter lists and two lambda ones); int 2; + // = 2; ; 2; => 2. + // + // Operands (n2 = 8, N2 = 10): L, N, M, P, a, b once each; x 2 + // and y 2 (the parameter and the lambda body). `var` is an + // unclassified `implicit_type` wrapper. + // + // Before the fix this read (10, 23, 8, 10): the two lambda + // keywords were billed nowhere. Billing them without listing + // `Static` / `Async` in `CsharpCode::is_primitive` reads + // (12, 25, 8, 10) — each keyword split across the kind-keyed + // and the lexeme-keyed operator map. + assert_halstead_counts::( + CSHARP_LAMBDA_MODIFIER_BOTH_SPELLINGS, + "foo.cs", + [10, 25, 8, 10], + "csharp lambda modifiers", + ); + + assert_modifier_spellings_are_one_operator( + CSHARP_LAMBDA_MODIFIER_BOTH_SPELLINGS, + &[ + ("static", Csharp::Static as u16), + ("async", Csharp::Async as u16), + ], + ); + } + #[cfg(feature = "go")] #[test] fn go_operators_and_operands() { @@ -4113,6 +4471,63 @@ mod tests { } } + #[cfg(feature = "perl")] + #[test] + fn perl_comment_leader_is_not_an_operator_1549() { + // tree-sitter-perl emits a comment's leading `#` as a `#` token + // under `comments`, and the getter listed it as an operator, so + // every comment billed one `#` (#1549). The oracle is the same + // source with its comments deleted, which must score identically. + // `$#a` and `qw#…#` are kept to pin that dropping `#` leaves the + // distinct `$#` operator and the `qw` delimiters alone. + // + // expected: operators `sub`, `{}`, `my`, `=`, `;` × 2, `return`, + // `$#` × 2, `+` → n1 = 8, N1 = 10. Operands `f`, `@a`, `x`, `y`, + // `a` × 2 → n2 = 5, N2 = 6. Before the fix the commented row + // added `#` → n1 = 9, N1 = 13. Once excluded, a comment feeds no + // axis to anchor on, so only the revert run guards the fixture: + // keep its comments. + for (source, label) in [ + ( + "sub f {\n # leading\n my @a = qw#x y#; # tail\n return $#a + $#a; # tail\n}\n", + "commented", + ), + ( + "sub f {\n my @a = qw#x y#;\n return $#a + $#a;\n}\n", + "comment-free twin", + ), + ] { + assert_halstead_counts::(source, "foo.pl", [8, 10, 5, 6], label); + } + } + + #[cfg(feature = "perl")] + #[test] + fn perl_bare_pattern_text_bills_no_operator_1549() { + // The bare `/…/` form keeps its pattern text in a + // `regex_pattern_content` that spells `#`, `[`, `]` and `\` as + // anonymous tokens, three of which an operator arm listed, so + // `/a#b[c]\#/` billed `#`, `[]` and `\` operators. `m//` and + // `qr//` emit no node for the same text, so they are the oracle: + // all three spellings must score as `/abc/` does in + // `perl_every_pattern_value_spelling_scores_alike`. + // + // expected per variant: operators `$` × 2, `=~` × 2, `and`, `;` + // → n1 = 4, N1 = 6; operands `$s` × 2 and the pattern × 2 → + // n2 = 2, N2 = 4. Before the fix the bare row scored n1 = 7, + // N1 = 14. The `#`, `[c]` and `\#` feed no axis once excluded, + // so nothing anchors them and only the revert run guards the + // fixture: keep them in the bare row. + for pattern in [r"/a#b[c]\#/", r"m/a#b[c]\#/", r"qr/a#b[c]\#/"] { + assert_halstead_counts::( + &format!("$s =~ {pattern} and $s =~ {pattern};\n"), + "foo.pl", + [4, 6, 2, 4], + &format!("pattern {pattern}"), + ); + } + } + #[cfg(feature = "perl")] #[test] fn perl_every_pattern_operation_spelling_scores_alike() { @@ -4343,6 +4758,73 @@ mod tests { ); } + /// Asserts a grammar's wrapped generic-list opener, `LT2`, never + /// reaches a node, so the `LT` its getter pairs as `<>` is the only + /// `<` a list opens with (#1559, grammar-dispatch §2). `lists` are the + /// lists the wrapped token opens; the positive control counts their + /// `LT` openers, so the assertion cannot pass on a fixture that + /// stopped producing those lists. + #[cfg(any(feature = "groovy", feature = "objc", feature = "rust"))] + #[track_caller] + fn assert_generic_opener_folds_to_lt( + source: &str, + [lt, lt2]: [u16; 2], + lists: &[&str], + openers: usize, + label: &str, + ) { + let parser = P::new(source.as_bytes().to_vec(), &PathBuf::from("foo"), None); + assert!( + !ast_has_kind_id(&parser, lt2), + "{label}::LT2 must stay collapsed to {label}::LT", + ); + let found = parser + .root() + .preorder() + .filter(|n| n.kind_id() == lt && n.parent().is_some_and(|p| lists.contains(&p.kind()))) + .count(); + assert_eq!( + found, openers, + "{label}: each generic-list opener must be an `LT`" + ); + } + + #[cfg(feature = "rust")] + #[test] + fn rust_generic_opener_alias_never_reaches_kind_id() { + assert_generic_opener_folds_to_lt::( + "fn u(x: Vec) -> impl Sized + use { x }\n", + [Rust::LT as u16, Rust::LT2 as u16], + &["type_arguments", "use_bounds"], + 2, + "Rust", + ); + } + + #[cfg(feature = "groovy")] + #[test] + fn groovy_generic_opener_alias_never_reaches_kind_id() { + assert_generic_opener_folds_to_lt::( + "class A { List v }\n", + [Groovy::LT as u16, Groovy::LT2 as u16], + &["type_parameters", "type_arguments"], + 2, + "Groovy", + ); + } + + #[cfg(feature = "objc")] + #[test] + fn objc_generic_opener_alias_never_reaches_kind_id() { + assert_generic_opener_folds_to_lt::( + "int f(void) { return g(NSArray); }\n", + [Objc::LT as u16, Objc::LT2 as u16], + &["argument_list"], + 1, + "Objc", + ); + } + #[cfg(feature = "perl")] #[test] fn perl_readline_brackets_are_operators() { @@ -5377,12 +5859,13 @@ end", "async def f():\n await a()\n await b()\n await c()\n", "foo.py", |metric| { - // expected operators: async, def, await (3 unique) - // await used three times -> N1 counts: async(1) def(1) await(3) = 5 + // expected operators: async, def, (), :, await (5 unique) + // N1: async(1) def(1) await(3) = 5, plus the four call + // and parameter-list `(` and the block `:` (#1486) = 10. // Before #413, Await + Await2 both matched, so `await` was a - // distinct operator twice: n1=4, N1=8. - assert_eq!(metric.halstead.unique_operators(), 3); - assert_eq!(metric.halstead.total_operators(), 5); + // distinct operator twice: one more unique and three more total. + assert_eq!(metric.halstead.unique_operators(), 5); + assert_eq!(metric.halstead.total_operators(), 10); }, ); } @@ -5394,10 +5877,11 @@ end", #[test] fn python_lambda_counted_once() { check_metrics::("g = lambda x: x + 1\n", "foo.py", |metric| { - // expected operators: =, lambda, + (3 unique, each used once) + // expected operators: =, lambda, :, + (4 unique, each used + // once; the parameter-list `:` since #1486). // Before #413, lambda was absent: only =, + were counted. - assert_eq!(metric.halstead.unique_operators(), 3); - assert_eq!(metric.halstead.total_operators(), 3); + assert_eq!(metric.halstead.unique_operators(), 4); + assert_eq!(metric.halstead.total_operators(), 4); }); } @@ -5411,11 +5895,12 @@ end", "match x:\n case 1:\n pass\n case _:\n pass\n", "foo.py", |metric| { - // expected operators: match, case, pass (3 unique) - // match(1) + case(2) + pass(2) = 5 total occurrences. + // expected operators: match, case, pass, : (4 unique) + // match(1) + case(2) + pass(2) + one block `:` per + // header (3, #1486) = 8 total occurrences. // Before #413, neither match nor case was counted (only pass). - assert_eq!(metric.halstead.unique_operators(), 3); - assert_eq!(metric.halstead.total_operators(), 5); + assert_eq!(metric.halstead.unique_operators(), 4); + assert_eq!(metric.halstead.total_operators(), 8); }, ); } @@ -5429,10 +5914,11 @@ end", "def f():\n global a\n nonlocal b\n", "foo.py", |metric| { - // expected operators: def, global, nonlocal (3 unique) + // expected operators: def, (), :, global, nonlocal + // (5 unique, each once; `()` and `:` since #1486) // Before #413, nonlocal was absent: only def, global counted. - assert_eq!(metric.halstead.unique_operators(), 3); - assert_eq!(metric.halstead.total_operators(), 3); + assert_eq!(metric.halstead.unique_operators(), 5); + assert_eq!(metric.halstead.total_operators(), 5); }, ); } @@ -5448,43 +5934,884 @@ end", "a not in b\na is not b\nnot c\nd in e\nf is g\nfor h in i:\n pass\n", "foo.py", |metric| { - // expected operators (7 unique): + // expected operators (8 unique): // "not in" (compound, once), "is not" (compound, once), // "not" (standalone `not c`, once), // "in" (standalone `d in e` + `for h in i` = twice), // "is" (standalone `f is g`, once), - // "for" (once), "pass" (once) - // Total occurrences: 1+1+1+2+1+1+1 = 8. + // "for" (once), "pass" (once), and the `for` header's + // block ":" (once, #1486) + // Total occurrences: 1+1+1+2+1+1+1+1 = 9. // Before #413, `a not in b` counted not+in (two) and // `a is not b` counted is+not (two); the compounds were // never classified. - assert_eq!(metric.halstead.unique_operators(), 7); - assert_eq!(metric.halstead.total_operators(), 8); + assert_eq!(metric.halstead.unique_operators(), 8); + assert_eq!(metric.halstead.total_operators(), 9); }, ); } - #[cfg(feature = "bash")] - #[test] - fn bash_operators_and_operands() { - check_metrics::( - "#!/bin/bash -f() { - local x=1 - if [ $x -eq 1 ]; then - echo 'one' - fi -}", - "foo.sh", - |metric| { - // Operators (9 unique, 9 occurrences): the opening - // delimiters `()`/`{}`/`[]` (each folded to one glyph and - // counted once per balanced pair, #695 — the closers no - // longer add a second operator), `local`, `=`, `if`, - // `then`, `fi`, `;`. - // Operands (6 unique, 7 occurrences): `f`, `x` (the - // assignment LHS `variable_name`, kind 160), `1` (twice: - // `=1` and `-eq 1`), `$x` (the `simple_expansion` — its + /// Asserts `count(halstead)` for each `(label, source, expected)` + /// row, one parse per row so a row's counts are its own. + #[cfg(any( + feature = "c", + feature = "cpp", + feature = "groovy", + feature = "java", + feature = "javascript", + feature = "kotlin", + feature = "mozcpp", + feature = "mozjs", + feature = "objc", + feature = "python", + feature = "rust", + feature = "typescript" + ))] + fn assert_halstead_rows( + file: &str, + rows: &[(&str, &str, [u64; 2])], + count: fn(&Stats) -> [u64; 2], + ) { + for (label, source, expected) in rows { + crate::test_support::check_func_space_only::( + source, + file, + &[crate::Metric::Halstead], + |space| { + assert_eq!( + count(&space.metrics.halstead), + *expected, + "{label}: {source:?}" + ); + }, + ); + } + } + + /// Asserts `[n1, N1]` for each row. + #[cfg(any( + feature = "c", + feature = "cpp", + feature = "groovy", + feature = "java", + feature = "javascript", + feature = "kotlin", + feature = "mozcpp", + feature = "mozjs", + feature = "objc", + feature = "python", + feature = "rust", + feature = "typescript" + ))] + fn assert_operator_rows(file: &str, rows: &[(&str, &str, [u64; 2])]) { + assert_halstead_rows::(file, rows, |h| [h.unique_operators(), h.total_operators()]); + } + + /// Asserts `[n2, N2]` for each row. + #[cfg(any(feature = "java", feature = "typescript"))] + fn assert_operand_rows(file: &str, rows: &[(&str, &str, [u64; 2])]) { + assert_halstead_rows::(file, rows, |h| [h.unique_operands(), h.total_operands()]); + } + + /// #1486: Python's bracket openers, `;` and `:` are vocabulary, as + /// they are in every other getter (#1395). Each row isolates one + /// position, so the expected `[n1, N1]` is derivable from the row + /// alone; a bracket pair bills its opener once, as `()` / `[]` / `{}`. + #[cfg(feature = "python")] + #[test] + fn python_punctuation_is_vocabulary_1486() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `()` + ("call", "f(x)\n", [1, 1]), + // `()`, `=` + ("keyword argument", "f(a=1)\n", [2, 2]), + // `()`, `*`, `,`, `**` + ("splat arguments", "f(*a, **k)\n", [4, 4]), + // `[]` + ("index", "a[1]\n", [1, 1]), + // `[]`, `:` twice + ("slice", "a[1:2:3]\n", [2, 3]), + // `{}`, `:` + ("dict", "{k: v}\n", [2, 2]), + // `{}`, `,` + ("set", "{1, 2}\n", [2, 2]), + // `[]`, `,` twice, `()` + ("list and tuple", "[1, (2, 3)]\n", [3, 4]), + // `()`, `for`, `in` + ("generator", "(x for x in y)\n", [3, 3]), + // `{}`, `:`, `for`, `in` + ("dict comprehension", "{k: v for k in y}\n", [4, 4]), + // `()`, `:=` — the walrus is its own token, not a `:` + ("walrus", "(n := 3)\n", [2, 2]), + // `lambda`, `:` + ("lambda", "lambda x: x\n", [2, 2]), + // `def`, `()`, `:` twice, `->`, `pass` + ("annotations", "def f(x: int) -> int:\n pass\n", [5, 6]), + // `:`, `=` + ("annotated assignment", "x: int = 1\n", [2, 2]), + // `@`, `def`, `()`, `:`, `pass` + ("decorator", "@d\ndef f():\n pass\n", [5, 5]), + // `if`, `:`, `pass` + ("if block", "if x:\n pass\n", [3, 3]), + // `while`, `:`, `pass` + ("while block", "while x:\n pass\n", [3, 3]), + // `for`, `in`, `:`, `pass` + ("for block", "for i in y:\n pass\n", [4, 4]), + // `with`, `:`, `pass` + ("with block", "with o:\n pass\n", [3, 3]), + // `try`, `except`, `:` twice, `pass` twice + ("try block", "try:\n pass\nexcept E:\n pass\n", [4, 6]), + // `class`, `:`, `pass` — the keyword is billed like `def` + // (#1550) + ("class block", "class C:\n pass\n", [3, 3]), + // `class` twice, `()`, `:` twice, `pass` — one `class` per + // keyword, nothing for the `ClassDefinition` wrapper + ( + "nested class", + "class C(B):\n class D:\n pass\n", + [4, 6], + ), + // `;` + ("statement separator", "a; b\n", [1, 1]), + ]; + assert_operator_rows::("foo.py", ROWS); + } + + /// #1486: an f-string replacement field's `{` and format-spec `:` are + /// spelling, not operations, like every other language's + /// interpolation opener (#1314) — the f-string rows bill nothing. + /// #1551 adds the self-documenting `=` (`f"{z=}"`), a directive of + /// the same field. The guard reads the token's *parent*, so a dict, + /// a keyword argument or a walrus written inside a replacement field + /// keeps its own operators. + #[cfg(feature = "python")] + #[test] + fn python_f_string_delimiters_are_not_operators_1486() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + ("interpolation opener", "f\"{x}\"\n", [0, 0]), + ("format spec colon", "f\"{x:>10}\"\n", [0, 0]), + ("nested format expression", "f\"{a:{w}}\"\n", [0, 0]), + // `{}`, `:`, `[]` of the dict and its subscript, nothing of + // the replacement field around them + ("dict inside a field", "f\"{ {k: v}[k] }\"\n", [3, 3]), + ("self-documenting =", "f\"{z=}\"\n", [0, 0]), + // the `=` keeps its spaces in the output; `!r` and `:` follow it + ("spaced = with conversion", "f\"{z = !r:>10}\"\n", [0, 0]), + // a `=` in a nested format expression (Python 3.12+) + ("nested self-documenting =", "f\"{a:{w=}}\"\n", [0, 0]), + // `()`, `=` — a keyword argument's `=` is a real operator + ("keyword argument inside a field", "f\"{g(a=1)}\"\n", [2, 2]), + // `()`, `:=` + ("walrus inside a field", "f\"{(y := 1)}\"\n", [2, 2]), + ]; + assert_operator_rows::("foo.py", ROWS); + } + + /// #1552: a JS-family `class` keyword bills as `function` does, once + /// per keyword. Shared by the four grammars of the macro; each runs + /// its own call so a grammar whose `Class2` stops being the keyword + /// leaf fails alone. Before #1552 every row billed one fewer. + #[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))] + const JS_FAMILY_CLASS_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}` + ("declaration", "class C {}\n", [2, 2]), + // `const`, `=`, `class`, `{}`, `;` — the class *expression* + // node wraps the keyword and bills nothing of its own + ("expression", "const D = class {};\n", [5, 5]), + // `class` twice, `extends`, `{}` twice + ("two classes", "class A {}\nclass B extends A {}\n", [3, 5]), + ]; + + #[cfg(any(feature = "javascript", feature = "mozjs"))] + #[test] + fn js_class_keyword_is_an_operator_1552() { + #[cfg(feature = "javascript")] + assert_operator_rows::("foo.js", JS_FAMILY_CLASS_ROWS); + #[cfg(feature = "mozjs")] + assert_operator_rows::("foo.js", JS_FAMILY_CLASS_ROWS); + } + + /// #1554: a function *expression* bills its `function` keyword once, + /// as a declaration does — the `function_expression` wrapper bills + /// nothing of its own. Before #1554 the plain and `async` rows billed + /// one extra unique and total operator, and the nested row one extra + /// unique and two extra totals. The declaration and generator rows + /// are controls: neither node was ever in the operator arm. + #[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))] + const JS_FAMILY_FUNCTION_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `function`, `()`, `{}` + ("declaration", "function h() {}\n", [3, 3]), + // `const`, `=`, `function`, `()`, `{}`, `;` + ("expression", "const f = function () {};\n", [6, 6]), + // `const`, `=`, `async`, `function`, `()`, `{}`, `;` + ( + "async expression", + "const f = async function () {};\n", + [7, 7], + ), + // `const`, `=`, `function`, `*`, `()`, `{}`, `;` — a generator + // expression is its own `generator_function` node, never billed + ( + "generator expression", + "const f = function* () {};\n", + [7, 7], + ), + // `const`, `=`, `return`, `;` x2, and `function`, `()`, `{}` + // twice each + ( + "nested expressions", + "const f = function () { return function () {}; };\n", + [7, 11], + ), + ]; + + #[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))] + #[test] + fn js_family_function_expression_bills_function_once_1554() { + #[cfg(feature = "javascript")] + assert_operator_rows::("foo.js", JS_FAMILY_FUNCTION_ROWS); + #[cfg(feature = "mozjs")] + assert_operator_rows::("foo.js", JS_FAMILY_FUNCTION_ROWS); + #[cfg(feature = "typescript")] + { + assert_operator_rows::("foo.ts", JS_FAMILY_FUNCTION_ROWS); + assert_operator_rows::("foo.tsx", JS_FAMILY_FUNCTION_ROWS); + } + } + + /// #1552 for the two TypeScript grammars: the shared `class` rows, + /// plus their own `interface` and `enum` declaration keywords. + #[cfg(feature = "typescript")] + #[test] + fn typescript_declaration_keywords_are_operators_1552() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `interface`, `{}` + ("interface", "interface I {}\n", [2, 2]), + // `enum`, `{}` + ("enum", "enum E { A }\n", [2, 2]), + ]; + assert_operator_rows::("foo.ts", JS_FAMILY_CLASS_ROWS); + assert_operator_rows::("foo.ts", ROWS); + assert_operator_rows::("foo.tsx", JS_FAMILY_CLASS_ROWS); + assert_operator_rows::("foo.tsx", ROWS); + } + + /// #1552: Java's type-declaration keywords are operators, as C#'s + /// are. The class-literal row pins the guard: `C.class` is one + /// operand, and its `class` leaf bills nothing of its own. + #[cfg(feature = "java")] + #[test] + fn java_declaration_keywords_are_operators_1552() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}` + ("class", "class C {}\n", [2, 2]), + // `interface`, `{}` + ("interface", "interface I {}\n", [2, 2]), + // `enum`, `{}` + ("enum", "enum E { A }\n", [2, 2]), + // `record`, `()`, `int`, `{}` + ("record", "record R(int x) {}\n", [4, 4]), + // `@interface`, `{}` + ("annotation type", "@interface N {}\n", [2, 2]), + // `class`, `{}`, `=`, `.`, `;` — one `class`: the literal's + // is part of the `C.class` operand + ("class literal", "class U { Object o = C.class; }\n", [5, 5]), + ]; + assert_operator_rows::("Foo.java", ROWS); + } + + /// #1560: a Java type name is an operand, as in C#, Rust, the C + /// family, Go, Groovy and TypeScript. Each row isolates one position + /// of the single `type_identifier` kind. The class-literal rows pin + /// that a type inside `Foo.class` is not billed again on top of the + /// literal's own operand, whether it sits there directly or under an + /// array or scoped type; the `var` rows, that the inferred-type + /// keyword bills nothing while a variable named `var` stays an + /// operand. + #[cfg(feature = "java")] + #[test] + fn java_type_names_are_operands_1560() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `C`, `I` x2, `Foo`, `f` + ( + "field type and implements", + "class C implements I { Foo f; }\ninterface I {}\n", + [4, 5], + ), + // `C`, `java`, `util`, `List`, `Foo`, `l` + ( + "scoped and generic", + "class C { java.util.List l; }\n", + [6, 6], + ), + // `C`, `T` x2, `Comparable` + ( + "type parameter", + "class C> {}\n", + [3, 4], + ), + // `C`, `m`, `Ex` + ("throws", "class C { void m() throws Ex {} }\n", [3, 3]), + // `C`, `Object`, `a`, `Foo.class` + ( + "class literal", + "class C { Object a = Foo.class; }\n", + [4, 4], + ), + // `C`, `Object`, `a`, `Foo[].class` + ( + "array class literal", + "class C { Object a = Foo[].class; }\n", + [4, 4], + ), + // `C`, `Object`, `a`, `java.util.List.class` + ( + "scoped class literal", + "class C { Object a = java.util.List.class; }\n", + [4, 4], + ), + // `C`, `m`, `x`, `1` — no `var` + ( + "inferred type", + "class C { void m() { var x = 1; } }\n", + [4, 4], + ), + // `C`, `m`, `var`, `1` — the variable's name + ( + "variable named var", + "class C { void m() { int var = 1; } }\n", + [4, 4], + ), + ]; + assert_operand_rows::("Foo.java", ROWS); + } + + /// #1552: Groovy's type-declaration keywords are operators, as + /// Java's are. `String.class` is a `field_access` whose `class` is a + /// plain identifier, so it stays an operand with no guard needed. + #[cfg(feature = "groovy")] + #[test] + fn groovy_declaration_keywords_are_operators_1552() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}` + ("class", "class C {}\n", [2, 2]), + // `interface`, `{}` + ("interface", "interface I {}\n", [2, 2]), + // `trait`, `{}` + ("trait", "trait T {}\n", [2, 2]), + // `enum`, `{}` + ("enum", "enum E { A }\n", [2, 2]), + // `record`, `()`, `{}` + ("record", "record R(int x) {}\n", [3, 3]), + // `def`, `=`, `.` — no `class` + ("class reference", "def a = String.class\n", [3, 3]), + // `=` twice: a script-level `record` / `trait` used as a name + // falls into error recovery with the keyword leaf, which is + // not a declaration and bills nothing + ("contextual names", "record = 3\ntrait = 4\n", [1, 2]), + ]; + assert_operator_rows::("foo.groovy", ROWS); + } + + /// #1552 rows shared by the two C++ clones. The template-parameter + /// rows pin the synonym: `class T` is `typename T`, and the two + /// spellings score alike. Both were unbilled until #1557 billed + /// `typename`, and the parameter `class` with it. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const CPP_DECLARATION_KEYWORD_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}`, `;` + ("class", "class C {};\n", [3, 3]), + // `struct`, `{}`, `;` + ("struct", "struct S {};\n", [3, 3]), + // `union`, `{}`, `int`, `;` twice + ("union", "union U { int a; };\n", [4, 5]), + // `enum`, `{}`, `;` + ("enum", "enum E { A };\n", [3, 3]), + // `enum`, `class`, `{}`, `;` + ("enum class", "enum class F { B };\n", [4, 4]), + // `struct`, `;` — an elaborated type bills its keyword + ("elaborated type", "struct S s;\n", [2, 2]), + // `template`, `<>`, `class`, `struct`, `{}`, `;` — the parameter + // list's brackets are one pair (#1559) + ( + "class parameter", + "template struct W {};\n", + [6, 6], + ), + // as above, `typename` in place of `class` + ( + "typename parameter", + "template struct W {};\n", + [6, 6], + ), + // `template`, `<>`, `class`, `=`, `int`, `struct`, `{}`, `;` + ( + "optional class parameter", + "template struct W {};\n", + [8, 8], + ), + // `template`, `<>`, `class`, `struct`, `{}`, `;` — `...` is an + // operand + ( + "variadic class parameter", + "template struct W {};\n", + [6, 6], + ), + ]; + + /// One call per clone, so reverting either arm fails its own row; + /// Mozcpp owns no file extension, so this is its only coverage. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_declaration_keywords_are_operators_1552() { + assert_operator_rows::("foo.cpp", CPP_DECLARATION_KEYWORD_ROWS); + assert_operator_rows::("foo.cpp", CPP_DECLARATION_KEYWORD_ROWS); + } + + /// #1552: C bills `struct` / `union` / `enum` as C++ does, so a + /// header scores the same whichever grammar parses it. + #[cfg(feature = "c")] + #[test] + fn c_declaration_keywords_are_operators_1552() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `struct`, `{}`, `int`, `;` twice + ("struct", "struct S { int a; };\n", [4, 5]), + // `union`, `{}`, `int`, `;` twice + ("union", "union U { int a; };\n", [4, 5]), + // `enum`, `{}`, `;` + ("enum", "enum E { A };\n", [3, 3]), + // `struct`, `;` + ("elaborated type", "struct S s;\n", [2, 2]), + ]; + assert_operator_rows::("foo.c", ROWS); + } + + /// #1552: ObjC bills C's `struct` and its own class and protocol + /// declaration keywords. `@class` already billed its `@` and stays + /// as it was. + #[cfg(feature = "objc")] + #[test] + fn objc_declaration_keywords_are_operators_1552() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `@interface`, `:` + ("interface", "@interface C : NSObject\n@end\n", [2, 2]), + // `@implementation` + ("implementation", "@implementation C\n@end\n", [1, 1]), + // `@protocol` + ("protocol", "@protocol P\n@end\n", [1, 1]), + // `struct`, `{}`, `int`, `;` twice + ("struct", "struct S { int a; };\n", [4, 5]), + // `@`, `;` — unchanged by #1552 + ("forward class", "@class Fwd;\n", [2, 2]), + ]; + assert_operator_rows::("foo.m", ROWS); + } + + /// #1556: Kotlin's `interface` bills as its `class` and `object` + /// do. The modifier rows score one above the bare keyword since + /// #1558: `enum` in `enum class` is a `class_modifier` like `data` + /// and `sealed`, and Kotlin bills its modifiers as C# does. + #[cfg(feature = "kotlin")] + #[test] + fn kotlin_declaration_keywords_are_operators_1556() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}` + ("class", "class C {}\n", [2, 2]), + // `interface`, `{}` + ("interface", "interface I {}\n", [2, 2]), + // `object`, `{}` + ("object", "object O {}\n", [2, 2]), + // `enum`, `class`, `{}` + ("enum class", "enum class E { A }\n", [3, 3]), + // `data`, `class`, `{}` + ("data class", "data class D {}\n", [3, 3]), + // `sealed`, `interface`, `{}` + ("sealed interface", "sealed interface S {}\n", [3, 3]), + // `interface`, `class`, `:`, `{}` twice + ( + "implementing class", + "interface J {}\nclass K : J {}\n", + [4, 5], + ), + ]; + assert_operator_rows::("foo.kt", ROWS); + } + + /// #1558: Kotlin bills its modifier keywords as C# bills its own, + /// one family per row, each keyword the leaf and never + /// its `*_modifier` wrapper. The "names" row is the control: the same + /// words used as names are operands, and add no operator. The rows + /// after it are names the grammar fails to parse, whose recovery keeps + /// the keyword leaf, and their control. + #[cfg(feature = "kotlin")] + #[test] + fn kotlin_modifiers_are_operators_1558() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `public`, `private`, `protected`, `internal`, `val` x4, + // `=` x4, `class`, `{}` + ( + "visibility", + "class C {\n public val a = 1\n private val b = 1\n \ + protected val c = 1\n internal val d = 1\n}\n", + [8, 14], + ), + // `abstract` x2, `open`, `final`, `override`, `class`, `{}` x2, + // `fun` x3, `()` x3, `=` + ( + "inheritance and member", + "abstract class A {\n abstract fun f()\n open fun g() {}\n \ + final override fun toString() = \"\"\n}\n", + [9, 15], + ), + // `inner`, `value`, `annotation`, `class` x3, `()`, `val`, `:` + ( + "class", + "inner class I\nvalue class V(val x: Int)\nannotation class N\n", + [7, 9], + ), + // `lateinit`, `companion`, `const`, `class`, `object`, `var`, + // `val`, `:`, `=`, `{}` x2 + ( + "property and companion", + "class C {\n lateinit var a: String\n companion object {\n \ + const val b = 1\n }\n}\n", + [10, 11], + ), + // six modifiers, `fun` x6, `()` x6, `{}` x5 + ( + "function", + "tailrec fun a() {}\noperator fun b() {}\ninfix fun c() {}\n\ + inline fun d() {}\nexternal fun e()\nsuspend fun f() {}\n", + [9, 23], + ), + // `inline`, `noinline`, `crossinline`, `vararg`, `fun`, + // `()` x3, `:` x3, `->` x2, `,` x2, `{}` + ( + "parameter", + "inline fun f(noinline a: () -> Unit, crossinline b: () -> Unit, \ + vararg c: Int) {}\n", + [10, 16], + ), + // `inline`, `reified`, `in`, `out`, `fun`, `class`, `<>` x2, + // `,`, `()`, `{}` + ( + "type parameter", + "inline fun f() {}\nclass Box\n", + [10, 11], + ), + // `expect`, `actual`, `fun` x2, `()` x2, `{}` + ("platform", "expect fun f()\nactual fun g() {}\n", [5, 7]), + // `fun`, `()`, `{}`: an `@N` use is the `annotation` node, not + // the keyword, so it bills no `annotation` operator + ("annotation use", "@N fun k() {}\n", [3, 3]), + // `val` x3, `=` x3: `open`, `out` and `value` are names here + ( + "names", + "val open = 1\nval out = 2\nval value = 3\n", + [2, 6], + ), + // The rows below are names the grammar fails to parse. Its + // recovery keeps the keyword leaf, which must bill nothing; + // each expected value is what the source scored before #1558. + // `class`, `{}` x2, `var`, `=` x2, `fun`, `()`: `open = !open` + // recovers into an `ERROR` holding `inheritance_modifier`. + ( + "name stranded in an ERROR", + "class Drawer {\n var open = false\n fun toggle() { open = !open }\n}\n", + [6, 8], + ), + // `val`: `suspend` never parses as a name. + ("suspend as a name", "val suspend = 1\n", [1, 1]), + // `fun`, `()`, `:` x2, `=`: a modifier list followed by an + // inserted zero-width parameter name. + ( + "parameter named vararg", + "fun f(vararg: Int): Int = vararg\n", + [4, 5], + ), + // The same row with a comment before the inserted name: an + // extra, so the name is still the next token. It scored + // [5, 6], a phantom `vararg` operator. + ( + "parameter named vararg, commented", + "fun f(vararg /*note*/: Int): Int = vararg\n", + [4, 5], + ), + // Control: a real `vararg` before a commented parameter keeps + // its operator. `vararg`, `fun`, `()`, `:`, `{}`. + ( + "vararg modifier, commented", + "fun f(vararg /*note*/ x: Int) {}\n", + [5, 5], + ), + // `class`, `()`, `:` + ("class parameter named open", "class C(open: Int)\n", [3, 3]), + // `val`, `=`, `{}`, `:` + ( + "lambda parameter named open", + "val l = { open: Int -> open }\n", + [4, 4], + ), + // Control for the gate: a real `suspend` in a function type's + // `type_modifiers` and in a declaration's `modifiers` bills + // twice. `val`, `:`, `suspend` x2, `()` x2, `->`, `=`, `{}` x2, + // `fun`. + ( + "suspend modifiers", + "val f: suspend () -> Unit = {}\nsuspend fun g() {}\n", + [8, 11], + ), + ]; + assert_operator_rows::("foo.kt", ROWS); + } + + /// #1558: `reified` is the one Kotlin modifier spelled as an aliased + /// named kind. `KotlinCode::is_primitive` keys it by its text, so + /// `bca ops` lists the word written, not the grammar's + /// `reification_modifier`. `n1` cannot see the difference — the kind + /// has one spelling either way — so the vocabulary is the only place + /// this is observable. + #[cfg(feature = "kotlin")] + #[test] + fn kotlin_reified_is_listed_by_its_text_1558() { + let ops = ops_of::("fun f() {}\n", "foo.kt"); + assert!( + ops.operators.iter().any(|op| op == "reified"), + "{:?}", + ops.operators + ); + assert!( + !ops.operators.iter().any(|op| op == "reification_modifier"), + "{:?}", + ops.operators + ); + } + + /// #1557 rows shared by the two C++ clones: the declaration keywords + /// #1552 left out, one per row, plus the two disambiguator positions + /// (`typename T::x`, `x.template f`) that bill the same leaf. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const CPP_MORE_DECLARATION_KEYWORD_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `namespace`, `{}` + ("namespace", "namespace N {}\n", [2, 2]), + // `typedef`, `int`, `;` + ("typedef", "typedef int T;\n", [3, 3]), + // `using`, `=`, `int`, `;` + ("alias declaration", "using U = int;\n", [4, 4]), + // `using`, `namespace`, `;` + ("using directive", "using namespace std;\n", [3, 3]), + // `template`, `<>`, `typename`, `;` and the `struct` of `S` + ( + "template declaration", + "template struct S;\n", + [5, 5], + ), + // `typename`, `::`, `;` + ("dependent type", "typename T::type x;\n", [3, 3]), + // `void`, `()` twice, `{}`, `.`, `template`, `<>`, `int`, `;` + ( + "template disambiguator", + "void g() { x.template f(); }\n", + [8, 9], + ), + ]; + + /// One call per clone, as for the #1552 rows. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_more_declaration_keywords_are_operators_1557() { + assert_operator_rows::("foo.cpp", CPP_MORE_DECLARATION_KEYWORD_ROWS); + assert_operator_rows::("foo.cpp", CPP_MORE_DECLARATION_KEYWORD_ROWS); + } + + /// #1557: `typedef` is a C keyword, so C and ObjC bill it as C++ + /// does — a header scores alike whichever grammar parses it. + #[cfg(any(feature = "c", feature = "objc"))] + const C_TYPEDEF_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `typedef`, `int`, `;` + ("typedef", "typedef int T;\n", [3, 3]), + // `typedef`, `struct`, `{}`, `int`, `;` twice + ("typedef struct", "typedef struct S { int a; } S;\n", [5, 6]), + ]; + + #[cfg(any(feature = "c", feature = "objc"))] + #[test] + fn c_family_typedef_is_an_operator_1557() { + #[cfg(feature = "c")] + assert_operator_rows::("foo.c", C_TYPEDEF_ROWS); + #[cfg(feature = "objc")] + assert_operator_rows::("foo.m", C_TYPEDEF_ROWS); + } + + /// #1557: the TypeScript declaration keywords #1552 left out. The + /// "keywords as names" row pins that each word, used as a *name*, + /// stays an operand: the grammar emits an `identifier` there. The two + /// exported-binding rows are the one place it emits the `type` leaf + /// for a name, which must bill nothing; the last two rows are their + /// controls, a type-only specifier and re-export whose `type` still + /// bills. + #[cfg(feature = "typescript")] + const TS_DECLARATION_KEYWORD_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `type`, `=`, `number`, `;` + ("type alias", "type T = number;\n", [4, 4]), + // `namespace`, `{}` + ("namespace", "namespace N {}\n", [2, 2]), + // `module`, `{}`. Billing the `module` declaration node instead + // of its keyword leaf would score this row the same: every + // spelling probed (bodiless, quoted, dotted, exported, ERROR + // recovery) emits the two together, so only the getter's + // choice of the leaf, not this row, keeps them from both billing. + ("module", "module M {}\n", [2, 2]), + // `declare`, `module`, `{}` + ("ambient module", "declare module \"m\" {}\n", [3, 3]), + // `declare`, `global`, `{}` + ("global augmentation", "declare global {}\n", [3, 3]), + // `import`, `type`, `{}`, `from`, `;` + ( + "type-only import", + "import type { A } from \"a\";\n", + [5, 5], + ), + // `let`, `=` twice, `;` twice, `.` — no keyword + ( + "keywords as names", + "let type = 1;\nmodule.exports = declare;\n", + [4, 6], + ), + // `const`, `=`, `;` twice, `export`, `{}` — no keyword. The + // grammar recovers an exported binding named `type` as the + // keyword leaf, under an `ERROR` or beside one in the specifier. + ( + "exported binding named type", + "const type = 1;\nexport { type };\n", + [5, 6], + ), + ( + "renamed exported binding named type", + "const type = 1;\nexport { type as kind };\n", + [5, 6], + ), + // `export`, `{}`, `type`, `from`, `;` — a type-only specifier + // parses cleanly, so its keyword still bills. + ( + "type-only export specifier", + "export { type Foo } from \"m\";\n", + [5, 5], + ), + // `export`, `type`, `*`, `from`, `;` — the pinned grammar fails + // TS 5.0's type-only re-export into an `ERROR` that hangs from the + // statement, not from an export clause, and its keyword bills. + ( + "type-only re-export", + "export type * from \"./types\";\n", + [5, 5], + ), + ]; + + /// #1557: a TypeScript type name is an operand, as C#, Rust and the + /// C family bill theirs. Each row's names are `type_identifier` + /// leaves except where noted, and before #1557 none of them billed. + #[cfg(feature = "typescript")] + const TS_TYPE_NAME_OPERAND_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `T` + ("alias name", "type T = number;\n", [1, 1]), + // `I` + ("interface name", "interface I {}\n", [1, 1]), + // `C` + ("class name", "class C {}\n", [1, 1]), + // `x` (an identifier), `Foo` + ("annotation", "let x: Foo;\n", [2, 2]), + // `m`, `Map`, `K` twice + ("generic", "let m: Map;\n", [3, 4]), + // `f`, `a` (identifiers), `U` twice + ("type parameter", "function f(a: U) {}\n", [3, 4]), + // `q`, `ns` (identifiers), `T` — the `nested_type_identifier` + // wrapper is unlisted, so its text is not a fourth operand + ("qualified type", "let q: ns.T;\n", [3, 3]), + // `C`, `I` + ("implements", "class C implements I {}\n", [2, 2]), + ]; + + #[cfg(feature = "typescript")] + #[test] + fn typescript_declaration_keywords_and_type_names_1557() { + assert_operator_rows::("foo.ts", TS_DECLARATION_KEYWORD_ROWS); + assert_operator_rows::("foo.tsx", TS_DECLARATION_KEYWORD_ROWS); + assert_operand_rows::("foo.ts", TS_TYPE_NAME_OPERAND_ROWS); + assert_operand_rows::("foo.tsx", TS_TYPE_NAME_OPERAND_ROWS); + } + + /// #1561: TypeScript's type-level operator keywords bill as their + /// value-level siblings `as`, `typeof` and `extends` do, each as its + /// keyword leaf under an unbilled wrapper. The `asserts a is Baz` row + /// pins that the `asserts` node and its keyword token bill once + /// between them, and the last row that the same words used as names + /// are operands. Both TypeScript grammars carry the change. + #[cfg(feature = "typescript")] + #[test] + fn typescript_type_operator_keywords_1561() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `let`, `=`, `satisfies`, `;` + ("satisfies", "let s = w satisfies Sat;\n", [4, 4]), + // `type`, `=`, `keyof`, `;` + ("keyof", "type K = keyof T;\n", [4, 4]), + // `type`, `<>` x2, `=`, `extends`, `infer`, `?`, `:`, `;` + ( + "infer", + "type I = X extends Array ? E : X;\n", + [8, 9], + ), + // `function`, `()`, `:` x2, `unknown`, `is`, `{}`, `return`, `;` + ( + "type predicate", + "function g(a: unknown): a is Baz { return true; }\n", + [8, 9], + ), + // `function`, `()`, `:` x2, `unknown`, `asserts`, `is`, `{}` + ( + "asserts predicate", + "function h(a: unknown): asserts a is Baz {}\n", + [7, 8], + ), + // `function`, `()`, `:` x2, `unknown`, `asserts`, `{}` + ( + "bare asserts", + "function k(a: unknown): asserts a {}\n", + [6, 7], + ), + // `const`, `=` and `;` five times each: the words are names + ( + "names", + "const keyof = 1;\nconst infer = 2;\nconst is = 3;\nconst asserts = 4;\n\ + const satisfies = 5;\n", + [3, 15], + ), + ]; + assert_operator_rows::("foo.ts", ROWS); + assert_operator_rows::("foo.tsx", ROWS); + } + + #[cfg(feature = "bash")] + #[test] + fn bash_operators_and_operands() { + check_metrics::( + "#!/bin/bash +f() { + local x=1 + if [ $x -eq 1 ]; then + echo 'one' + fi +}", + "foo.sh", + |metric| { + // Operators (9 unique, 9 occurrences): the opening + // delimiters `()`/`{}`/`[]` (each folded to one glyph and + // counted once per balanced pair, #695 — the closers no + // longer add a second operator), `local`, `=`, `if`, + // `then`, `fi`, `;`. + // Operands (6 unique, 7 occurrences): `f`, `x` (the + // assignment LHS `variable_name`, kind 160), `1` (twice: + // `=1` and `-eq 1`), `$x` (the `simple_expansion` — its // inner `variable_name` leaf is now suppressed so `$x` // counts once, #695), `echo`, `'one'`. // N2 was 8 before #1351: `echo` counted twice, once as the @@ -9365,13 +10692,14 @@ f() { #[cfg(feature = "objc")] #[test] fn objc_nsstring_literal_is_one_operand() { - // expected: [n1, N1, n2, N2]. Before the guard the first two rows - // read [8, 8, 5, 5] and [8, 9, 6, 6] — one `@` operator per - // literal; the boxing control is unchanged. + // expected: [n1, N1, n2, N2], each operator count including the + // wrapper's `@implementation` (#1552). Before the guard the first + // two rows read [9, 9, 5, 5] and [9, 10, 6, 6] — one `@` operator + // per literal; the boxing control is unchanged. let cases = [ - ("NSString *s = @\"str\";", [7, 7, 5, 5]), - ("NSString *t = @\"x\" @\"y\";", [7, 7, 6, 6]), - ("NSNumber *n = @42;", [8, 8, 5, 5]), + ("NSString *s = @\"str\";", [8, 8, 5, 5]), + ("NSString *t = @\"x\" @\"y\";", [8, 8, 6, 6]), + ("NSNumber *n = @42;", [9, 9, 5, 5]), ]; for (body, counts) in cases { let source = format!("@implementation Foo\n- (void)m {{\n {body}\n}}\n@end\n"); @@ -9405,32 +10733,33 @@ f() { @end "; check_metrics::(source, "foo.m", |metric| { - // n1 = 14 unique operators: + // n1 = 15 unique operators: // `&&`, `()`, `+`, `-`, `:`, `;`, `<`, `=`, `>`, - // `[]` (message send), `if`, `int`, `return`, `{}`. + // `[]` (message send), `if`, `int`, `return`, `{}`, + // `@implementation` (#1552). // The `@` of `@"hi"` is part of the literal's operand // key, not an operator (see `objc_nsstring_literal_is_one_operand`). // n2 = 10 unique operands: // `Foo`, `bar`, `log`, `self`, `x`, `y`, `0`, `1`, `10`, // `@"hi"` (the ObjC string literal). - assert_eq!(metric.halstead.unique_operators(), 14); + assert_eq!(metric.halstead.unique_operators(), 15); assert_eq!(metric.halstead.unique_operands(), 10); insta::assert_json_snapshot!(metric.halstead, @r#" { - "unique_operators": 14, - "total_operators": 22, + "unique_operators": 15, + "total_operators": 23, "unique_operands": 10, "total_operands": 14, - "length": 36, - "estimated_program_length": 86.52224985768008, - "purity_ratio": 2.403395829380002, - "vocabulary": 24, - "volume": 165.0586500259616, - "difficulty": 9.8, - "level": 0.1020408163265306, - "effort": 1617.5747702544238, - "time": 89.86526501413465, - "bugs": 0.04593266617952463 + "length": 37, + "estimated_program_length": 91.82263988300141, + "purity_ratio": 2.481692969810849, + "vocabulary": 25, + "volume": 171.8226790216648, + "difficulty": 10.5, + "level": 0.09523809523809523, + "effort": 1804.1381297274804, + "time": 100.22989609597113, + "bugs": 0.049399808887691035 } "#); }); @@ -9704,15 +11033,15 @@ f() { // Operators, keyed by kind_id except the text-keyed primitives: // `{` x3 (class body, `m1`, `m2`), `int` x3, `;` x4 (the field, // the two returns, the struct terminator), `(` x2, `return` x2, - // `->` x2, and the one `*` of `S* p`. `struct` is in no arm, and - // #695 dropped every closing delimiter. n1 = 7, N1 = 17. + // `->` x2, the one `*` of `S* p`, and `struct` (#1552). #695 + // dropped every closing delimiter. n1 = 8, N1 = 18. // // Operands, keyed by source text: `S` x2 (the name and the // parameter type), `x` x3, `m1`, `m2`, `p` x2, and — the fix — // `this` x1. n2 = 6, N2 = 10. Before #1361: n2 5, N2 9, with // every operator count identical, which is the whole claim. let counts = format!("{label}: this->x and p->x are the same shape"); - assert_halstead_counts::(CPP_THIS_RECEIVER_PARITY, file, [7, 17, 6, 10], &counts); + assert_halstead_counts::(CPP_THIS_RECEIVER_PARITY, file, [8, 18, 6, 10], &counts); assert_ops_operands::( CPP_THIS_RECEIVER_PARITY, file, @@ -9721,6 +11050,354 @@ f() { ); } + /// Every C++ overloaded-operator name the grammar spells as a token + /// of its own, plus the `co_await` expression and the conversion + /// operator's `operator` keyword (#1296). + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const CPP_OPERATOR_NAMES: &str = + "struct S { int operator[](int); void operator()(); bool operator co_await(); }; +long operator\"\"_x(unsigned long long); +S::operator bool() const; +S s; int a = s[1]; auto t = co_await s; +"; + + /// Asserts `CPP_OPERATOR_NAMES` bills each operator name, keyword and + /// `co_await` as an operator under parser `T`. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + #[track_caller] + fn assert_cpp_operator_names(label: &str) { + // Operators (n1 = 17, N1 = 40): `operator` 5 (four + // `operator_name`s and the `operator_cast`); the names `[]` 1, + // `()` 1, `""` 1; `co_await` 2 (the name and the expression); + // the `(` of a parameter list 5; `;` 9; `int` 3; `long` 3; + // `bool` 2; `=` 2; `{` 1; `void` 1; `unsigned` 1; `::` 1; the + // subscript `[` 1; `struct` 1 (#1552). `auto` and `const` are in + // no arm. + // + // Operands (n2 = 6, N2 = 9): `s` 3, `S` 2 (the qualifier in + // `S::operator bool` is a namespace identifier, which #1096 + // leaves unbilled), `_x`, `a`, `1`, `t`. + // + // Before #1296: (12, 30, 6, 9) with `struct`, the five operator + // kinds billed nowhere. + assert_halstead_counts::(CPP_OPERATOR_NAMES, "ops.cpp", [17, 40, 6, 9], label); + + // `operator[]` / `operator()` are kinds of their own, so they sit + // beside the subscript `[` and the parameter-list `(` as two more + // operators, each rendered as the operator it names so that every + // listed entry is distinct. + let ops = ops_of::(CPP_OPERATOR_NAMES, "ops.cpp"); + for (name, entries) in [ + ("operator", 1), + ("co_await", 1), + ("\"\"", 1), + ("operator[]", 1), + ("operator()", 1), + ("[]", 1), + ("()", 1), + ] { + assert_eq!( + ops.operators.iter().filter(|o| *o == name).count(), + entries, + "{label}: `{name}` must be {entries} operator entry; operators were {:?}", + ops.operators + ); + assert!( + !ops.operands.iter().any(|o| o == name), + "{label}: `{name}` must not be an operand; operands were {:?}", + ops.operands + ); + } + } + + /// Regression for #1296: a C++ operator name is a Halstead operator, + /// as `operator +`'s `+` already was. One row per clone, so reverting + /// either arm fails its own row (grammar-dispatch section 11); Mozcpp + /// owns no file extension, so this row is its only coverage. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_operator_names_are_operators() { + assert_cpp_operator_names::("cpp"); + assert_cpp_operator_names::("mozcpp"); + } + + /// Asserts a coroutine's `co_yield` and `co_return` keywords bill as + /// operators under parser `T`, as `return` does (#1547). + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + #[track_caller] + fn assert_cpp_coroutine_keywords(label: &str) { + let source = "T gen() { co_yield 1; co_yield 2; co_return 3; }"; + // Operators (n1 = 5, N1 = 8): `()` 1, `{}` 1, `co_yield` 2, + // `co_return` 1, `;` 3. Operands (n2 = 5, N2 = 5): `T`, `gen`, + // `1`, `2`, `3`. Before #1547: (3, 5, 5, 5), both keywords + // billed nowhere. + assert_halstead_counts::(source, "foo.cpp", [5, 8, 5, 5], label); + let ops = ops_of::(source, "foo.cpp"); + for keyword in ["co_yield", "co_return"] { + assert!( + ops.operators.iter().any(|o| o == keyword), + "{label}: `{keyword}` must be an operator; operators were {:?}", + ops.operators + ); + } + } + + /// Regression for #1546: a zero-width node is billed in neither + /// Halstead half. Error recovery inserts one for a token the source + /// lacks, usually flagged MISSING, and every getter that classifies + /// the kind billed it — an identifier as the empty-string operand. + /// One fixture per language, each a parse on which recovery inserts + /// an operand-classified zero-width node; before the fix every row + /// billed `""`. C#'s and Elixir's are not flagged MISSING, which is + /// why the guard keys on width: an `is_missing()` guard leaves those + /// two rows billing `""`. Each row names its zero-width operand's + /// kind and flag, so the two unflagged rows keep discriminating. The + /// rows pin no counts, since the recovery trees are the grammars' to + /// change (grammar-dispatch §6); the operator half, an inserted `;` or + /// `)`, is anchored by `cpp_missing_tokens_in_valid_code_are_not_billed`. + #[cfg(any( + feature = "bash", + feature = "c", + feature = "cpp", + feature = "csharp", + feature = "elixir", + feature = "go", + feature = "groovy", + feature = "irules", + feature = "java", + feature = "javascript", + feature = "kotlin", + feature = "lua", + feature = "mozcpp", + feature = "mozjs", + feature = "objc", + feature = "perl", + feature = "php", + feature = "python", + feature = "ruby", + feature = "rust", + feature = "tcl", + feature = "typescript", + ))] + #[test] + fn zero_width_nodes_bill_neither_halstead_half() { + // `(language, file, source, kind, is_missing)`: the last two name + // the zero-width operand each fixture's recovery inserts, so a + // grammar bump that recovers some other way fails the row rather + // than passing it on an unrelated zero-width node. + let cases: &[(LANG, &str, &str, &str, bool)] = &[ + #[cfg(feature = "bash")] + (LANG::Bash, "foo.sh", "echo a $( )\n", "word", true), + #[cfg(feature = "c")] + ( + LANG::C, + "foo.c", + "int x; void () { x = 1; }\n", + "identifier", + true, + ), + #[cfg(feature = "cpp")] + ( + LANG::Cpp, + "foo.cpp", + "void f(S *s) {\n s->~D();\n}\n", + "identifier", + true, + ), + #[cfg(feature = "csharp")] + ( + LANG::Csharp, + "foo.cs", + "class A { int = 1; }\n", + "identifier", + false, + ), + #[cfg(feature = "elixir")] + (LANG::Elixir, "foo.ex", "x = a.\n", "identifier", false), + #[cfg(feature = "go")] + ( + LANG::Go, + "foo.go", + "package m\nfunc f() { a.() }\n", + "type_identifier", + true, + ), + #[cfg(feature = "groovy")] + ( + LANG::Groovy, + "foo.groovy", + "def x = a.\n", + "identifier", + true, + ), + #[cfg(feature = "irules")] + ( + LANG::Irules, + "foo.irule", + "when HTTP_REQUEST { set x [expr {1 + }] }\n", + "number", + true, + ), + #[cfg(feature = "java")] + ( + LANG::Java, + "Foo.java", + "class A { int = 1; }\n", + "identifier", + true, + ), + #[cfg(feature = "javascript")] + ( + LANG::Javascript, + "foo.js", + "let {a: } = b;\n", + "identifier", + true, + ), + #[cfg(feature = "kotlin")] + ( + LANG::Kotlin, + "foo.kt", + "fun f(: Int) {}\n", + "identifier", + true, + ), + #[cfg(feature = "lua")] + (LANG::Lua, "foo.lua", "x = a.\n", "identifier", true), + #[cfg(feature = "mozcpp")] + ( + LANG::Mozcpp, + "foo.cpp", + "void f(S *s) {\n s->~D();\n}\n", + "identifier", + true, + ), + #[cfg(feature = "mozjs")] + ( + LANG::Mozjs, + "foo.jsm", + "let {a: } = b;\n", + "identifier", + true, + ), + #[cfg(feature = "objc")] + ( + LANG::Objc, + "foo.m", + "@interface A\n- (void)f:;\n@end\n", + "identifier", + true, + ), + #[cfg(feature = "perl")] + (LANG::Perl, "foo.pl", "sub { $x = ; }\n", "identifier", true), + #[cfg(feature = "php")] + (LANG::Php, "foo.php", ";\n", "name", true), + #[cfg(feature = "python")] + ( + LANG::Python, + "foo.py", + "for in x:\n pass\n", + "identifier", + true, + ), + #[cfg(feature = "ruby")] + (LANG::Ruby, "foo.rb", "x = a.\n", "identifier", true), + #[cfg(feature = "rust")] + ( + LANG::Rust, + "foo.rs", + "fn f() { let x = a.; }\n", + "field_identifier", + true, + ), + #[cfg(feature = "tcl")] + (LANG::Tcl, "foo.tcl", "puts [expr {1 + }]\n", "number", true), + #[cfg(feature = "typescript")] + ( + LANG::Typescript, + "foo.ts", + "let {a: } = b;\n", + "identifier", + true, + ), + #[cfg(feature = "typescript")] + (LANG::Tsx, "foo.tsx", "let {a: } = b;\n", "identifier", true), + ]; + crate::test_support::assert_fixtures_present(cases); + + let mut billed = Vec::new(); + for (lang, file, source, kind, missing) in cases { + let ast = crate::test_support::parse_named(*lang, file, source); + assert!( + ast.root_node().preorder().any(|node| { + node.start_byte() == node.end_byte() + && node.kind() == *kind + && node.is_missing() == *missing + }), + "{lang:?}: `{source}` no longer recovers with a zero-width `{kind}` \ + (is_missing: {missing}); find another fixture" + ); + let ops = ast.ops().expect("ops walk must yield a top-level Ops"); + assert!( + !ops.operands.is_empty(), + "{lang:?}: `{source}` billed no operand at all" + ); + billed.extend(ops.operands.iter().any(String::is_empty).then_some(*lang)); + } + assert!( + billed.is_empty(), + "a zero-width node was billed as the empty operand in {billed:?}" + ); + } + + /// Asserts two valid C++ programs that tree-sitter-cpp 0.23.4 can only + /// parse by inserting MISSING tokens bill only the tokens written + /// (#1546), under parser `T`. Both come from the DeepSpeech corpus. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + #[track_caller] + fn assert_cpp_missing_tokens_unbilled(label: &str) { + // `openfst/dfs-visit.h:75`: an explicit destructor call through a + // template name. Recovery reads `s->~D`, + // `~`, `<`, `>`, `;`. Operands (n2 = 5, N2 = 6): `f`, `S`, `s` 2, + // `D`, `F`. Before #1546: (9, 10, 6, 7), the extra being `""`. + let destructor = "void f(S *s) {\n s->~D();\n}\n"; + assert_halstead_counts::(destructor, "foo.cpp", [9, 10, 5, 6], label); + let ops = ops_of::(destructor, "foo.cpp"); + assert_eq!(ops.operands, ["D", "F", "S", "f", "s"], "{label}"); + + // `kenlm/lm/search_trie.cc:286`: a `typename` functional cast + // used as a statement. Recovery inserts a MISSING `)` after + // `quant_` and a MISSING `;` after `order`. Operators (n1 = 9, + // N1 = 11): `void`, `()` 3, `{}`, `::`, `,`, `-`, `.`, the one + // written `;`, and `typename` (#1557). Operands (n2 = 7, N2 = 7): + // `f`, `MiddlePointer`, `quant_`, `order`, `2`, `Write`, `w`. + // Before #1546 the MISSING `;` was a second `;`. + let cast = "void f() {\n typename Quant::MiddlePointer(quant_, order - 2).Write(w);\n}\n"; + assert_halstead_counts::(cast, "foo.cpp", [9, 11, 7, 7], label); + } + + /// Regression for #1546 on the two C++ clones. Mozcpp owns no file + /// extension, so this row is its only coverage. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_missing_tokens_in_valid_code_are_not_billed() { + assert_cpp_missing_tokens_unbilled::("cpp"); + assert_cpp_missing_tokens_unbilled::("mozcpp"); + } + + /// Regression for #1547: `co_return` and `co_yield` are Halstead + /// operators. One row per clone, so reverting either arm fails its + /// own row; Mozcpp owns no file extension, so this row is its only + /// coverage. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_coroutine_keywords_are_operators() { + assert_cpp_coroutine_keywords::("cpp"); + assert_cpp_coroutine_keywords::("mozcpp"); + } + /// Regression for #1361: a C++ `this` is a Halstead operand. /// /// `Cpp::This` / `Mozcpp::This` were in neither arm of diff --git a/src/metrics/mi.rs b/src/metrics/mi.rs index 67f26815..85b2b104 100644 --- a/src/metrics/mi.rs +++ b/src/metrics/mi.rs @@ -225,9 +225,9 @@ mod tests { metric.mi, @r#" { - "original": 151.2033158832232, - "sei": 142.64306171748976, - "visual_studio": 88.42299174457497 + "original": 146.56146936754593, + "sei": 135.94629276875443, + "visual_studio": 85.70846161844791 } "# ); diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index b377aa90..1d76d550 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -33,6 +33,11 @@ pub mod tokens; /// Weighted Methods per Class. pub mod wmc; +mod elixir_decisions; + +#[cfg(test)] +#[path = "bash_test_connective_tests.rs"] +mod bash_test_connective_tests; // Gated on the union of every feature any test in the module names — // the `FIXTURES` rows plus the `bash` / `lua` / `c` row set that // `a_language_with_no_member_construct_emits_neither_block` uses. The @@ -69,6 +74,12 @@ mod container_scope_tests; #[cfg(any(feature = "cpp", feature = "mozcpp"))] #[path = "cpp_alternative_tokens_tests.rs"] mod cpp_alternative_tokens_tests; +#[cfg(test)] +#[path = "elixir_named_operator_tests.rs"] +mod elixir_named_operator_tests; +#[cfg(test)] +#[path = "perl_keyword_key_tests.rs"] +mod perl_keyword_key_tests; /// Divides a metric sum by a count, guarding the divisor with `.max(1)`. /// diff --git a/src/metrics/nexits.rs b/src/metrics/nexits.rs index 3c355a78..347c1a49 100644 --- a/src/metrics/nexits.rs +++ b/src/metrics/nexits.rs @@ -181,8 +181,29 @@ impl_exit_match_kinds!( Tsx, [ReturnStatement, ThrowStatement, YieldExpression] ); -impl_exit_match_kinds!(CppCode, Cpp, [ReturnStatement, ThrowStatement]); -impl_exit_match_kinds!(MozcppCode, Mozcpp, [ReturnStatement, ThrowStatement]); +// A coroutine's `co_return` is its `return`, and `co_yield` suspends it +// back to the caller as a Python or JS `yield` does, so both count +// (#1547). +impl_exit_match_kinds!( + CppCode, + Cpp, + [ + ReturnStatement, + ThrowStatement, + CoReturnStatement, + CoYieldStatement + ] +); +impl_exit_match_kinds!( + MozcppCode, + Mozcpp, + [ + ReturnStatement, + ThrowStatement, + CoReturnStatement, + CoYieldStatement + ] +); // C has no exceptions: `return` is the only exit kind (no `throw`). impl_exit_match_kinds!(CCode, C, [ReturnStatement]); // Objective-C adds `@throw` on top of C's `return` (the `throw_statement` @@ -2900,6 +2921,30 @@ end", ); } + /// A coroutine with two `co_yield`s and a `co_return`, which count + /// as three exits, as Python's two `yield`s and a `return` do (#1547). + /// Before #1547 neither statement counted and the sum was 0. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const CPP_COROUTINE: &str = "T gen() { co_yield 1; co_yield 2; co_return; }"; + + // One test per clone, so reverting either impl fails its own test. + #[cfg(feature = "cpp")] + #[test] + fn cpp_co_return_and_co_yield_count_as_exits() { + check_metrics::(CPP_COROUTINE, "foo.cpp", |metric| { + assert_eq!(metric.nexits.nexits_sum(), 3); + }); + } + + // Mozcpp owns no file extension, so this is its only coverage. + #[cfg(feature = "mozcpp")] + #[test] + fn mozcpp_co_return_and_co_yield_count_as_exits() { + check_metrics::(CPP_COROUTINE, "foo.cpp", |metric| { + assert_eq!(metric.nexits.nexits_sum(), 3); + }); + } + #[cfg(feature = "python")] #[test] fn python_yield_counts_as_exit() { diff --git a/src/metrics/npa/shared.rs b/src/metrics/npa/shared.rs index 3a46064f..cbd85780 100644 --- a/src/metrics/npa/shared.rs +++ b/src/metrics/npa/shared.rs @@ -668,328 +668,6 @@ pub(crate) fn ruby_in_clause_counts(in_clause: &Node, source: &[u8]) -> bool { }) } -/// Whether `node` is a `when` `binary_operator` — the shape a repeated -/// guard's alternatives nest through. -/// -/// `binary_operator` carries three kind aliases at this pin so it is -/// matched by rule name rather than by enumerating ids -/// (grammar-dispatch §1); `when` has exactly one id. -fn elixir_is_when_operator(node: &Node) -> bool { - const BINARY_OPERATOR: &str = "binary_operator"; - - node.kind() == BINARY_OPERATOR - && node - .child_by_field_name("operator") - .is_some_and(|operator| operator.kind_id() == Elixir::When as u16) -} - -/// The one alternative a guard's `when` token introduces. -/// -/// `head when a when b` parses right-associatively as -/// `head when (a when b)`, so each `when` operator owns exactly one -/// alternative: the nested operator's `left` where its `right` nests a -/// further `when`, and its own `right` otherwise. One alternative per -/// token is what holds the `Abc` slot count level with the `Cyclomatic` -/// decision count at every chain length (§8), and what stops a chain of -/// n alternatives being counted once per token that can see it (§5). -/// -/// The `?` is infallible at the pinned grammar — `binary_operator` -/// declares `right` required — and is spelled as an `Option` because -/// `AGENTS.md` bans `expect` outside tests. -pub(crate) fn elixir_when_alternative<'a>(when_operator: &Node<'a>) -> Option> { - let right = when_operator.child_by_field_name("right")?; - if elixir_is_when_operator(&right) { - right.child_by_field_name("left") - } else { - Some(right) - } -} - -/// Whether a `when` operator token spells a real guard — a function -/// head's (`def f(x) when g do`) or a clause's (`x when g -> …`) — -/// rather than a typespec's `when` binding clause -/// (`@spec f(a) :: a when a: integer`), which is type syntax and no -/// decision at all. -/// -/// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode` so the -/// two cannot disagree about what a guard is (grammar-dispatch §7). -/// Elixir has no dedicated guard production — `x when g` is an ordinary -/// `binary_operator` — so the position it sits in is the only thing that -/// tells a guard from a typespec, and the allowlist below is that -/// position set at the pinned grammar: the `left` slot of a -/// `stab_clause` (`case` / `cond` / `fn` / `receive` / `with`'s `else` -/// / `try`'s handlers), or an argument of a definition Call that takes -/// a guarded head. -/// -/// `arguments` carries five kind aliases at this pin and -/// `binary_operator` three, so both are matched by rule name rather -/// than by enumerating ids (grammar-dispatch §1). -/// -/// Repeated guards (`when a when b`) are an or-chain: Elixir tries each -/// `when` expression in turn and moves to the next when the previous one -/// is false *or raises*, so the construct carries one decision per -/// alternative, level with the `when a or b` spelling. They parse -/// right-associatively into nested `when` operators, and only the -/// outermost sits on the anchor — so a nested one is a guard too, and -/// the climb below walks the `when` operators between it and the anchor -/// before asking the position question. [`elixir_when_alternative`] -/// names the one alternative each token introduces, which is how the -/// matching `Abc` slot stays one-per-token rather than double counting -/// the chain (grammar-dispatch §5). -/// -/// The climb stops at the first non-`when` ancestor, so an ordinary -/// single guard pays no extra step and a chain pays one per alternative -/// it lists — a bound set by the guard, not by the tree's depth. -/// -/// Both `return false` guards below are unreachable at the pin, not -/// untested: a `when` *token*'s chain always holds at least the -/// `binary_operator` it belongs to, and that operator always sits under -/// something, because the Elixir root is `source` and no `when` -/// operator can be it. They stay as defensive `else` arms because -/// `AGENTS.md` bans the `expect` that would replace them. -pub(crate) fn elixir_when_is_guard<'a>( - node: &Node<'a>, - code: &'a [u8], - ancestors: Ancestors<'a, '_>, -) -> bool { - use Elixir as E; - - const ARGUMENTS: &str = "arguments"; - - let mut chain = ancestors.iter(node); - // The token's parent is the `when` operator node itself; the first - // ancestor above the chain of `when` operators is the position that - // decides. - let Some((mut operator, _)) = chain.next() else { - return false; - }; - let mut above = chain.next(); - while let Some((enclosing, _)) = above.filter(|(ancestor, _)| elixir_is_when_operator(ancestor)) - { - operator = enclosing; - above = chain.next(); - } - let Some((parent, _)) = above else { - return false; - }; - if parent.kind_id() == E::StabClause as u16 { - return parent - .child_by_field_name("left") - .is_some_and(|left| left.id() == operator.id()); - } - parent.kind() == ARGUMENTS - && chain.next().is_some_and(|(call, _)| { - crate::lang_helpers::elixir::elixir_call_keyword(&call, code).is_some_and(|keyword| { - crate::lang_helpers::elixir::elixir_is_method_macro(keyword) - || matches!(keyword, "defguard" | "defguardp") - }) - }) -} - -/// Returns the sole pattern of a `stab_clause` whose left-hand side -/// carries no `when` guard, or `None` otherwise. -/// -/// The grammar's `left` field is an `arguments` node for a plain -/// pattern list, but a guarded clause (`_ when g ->`) re-shapes it -/// into a `binary_operator` wrapping the patterns and the guard — so -/// checking the field's kind answers "is there a guard" for free. The -/// kind is compared as a string because the grammar aliases several -/// internal rules to `arguments` with distinct kind ids -/// (`Arguments2`..`Arguments5`); grammar-dispatch §1 prefers the one -/// string comparison over enumerating them. A zero-arity clause carries -/// no `left` field at all (`fn -> … end`), which the same `filter` folds -/// into "no plain pattern list" — the two spellings of "nothing to -/// inspect" are one question, so they share one exit. Named-children -/// filtering skips the anonymous `(` `)` tokens a parenthesised clause -/// head carries, so an empty pair (`fn () -> … end`) also yields zero -/// patterns. Clauses with zero or several patterns return `None`. -pub(crate) fn elixir_sole_unguarded_pattern<'a>(node: &Node<'a>) -> Option> { - let left = node - .child_by_field_name("left") - .filter(|left| left.kind() == "arguments")?; - let mut patterns = left.children().filter(Node::is_named); - let sole = patterns.next()?; - patterns.next().is_none().then_some(sole) -} - -/// Returns `true` when `node` is a construct's free default clause: -/// a bare `_ ->` catch-all outside an anonymous fn, or an unguarded -/// `true ->` directly under a `cond`. -/// -/// Both shapes hinge on the clause's sole unguarded pattern, and -/// [`elixir_sole_unguarded_pattern`]'s child scan allocates a cursor -/// (the #1112 malloc) — so the pattern is extracted once here and the -/// two shapes branch on its (kind, text), instead of every counted -/// `stab_clause` paying the extraction twice through two independent -/// predicates. -/// -/// Bare `_ ->`: Elixir has no dedicated wildcard token — `_` parses -/// as an ordinary `identifier` — so the bytes decide (grammar-dispatch -/// §10). A named discard (`_x ->`) binds a value the body can read -/// and keeps counting, matching Rust's bare-`_`-only `MatchArm` rule; -/// guarded wildcards never reach the text check because -/// [`elixir_sole_unguarded_pattern`] rejects them. The exclusion does -/// NOT apply when the clause's container is an `anonymous_function`: -/// a multi-clause `fn` is a dispatch like `case` — n clauses are n−1 -/// decisions — and its free base path is already granted by the -/// head-clause skip (#776), so excluding a trailing `_ ->` too would -/// leave `fn 0 -> :a; _ -> :b end` at zero decisions while the -/// identical `case` reports one. -/// -/// `true ->` under `cond`: the exclusion is *shape-based* — any -/// unguarded `true ->` whose parent is the `do_block` of a `Call` -/// spelling `cond`, whatever the arm's position. That deliberately -/// matches the sibling convention: Rust's bare-`_` `MatchArm` -/// exclusion is equally position-blind. The same clause under `case` -/// is an ordinary boolean pattern match, so the exclusion is anchored -/// to the owning construct (grammar-dispatch §8); a guarded -/// `true when g ->` is a real decision and never reaches the -/// container check. -fn elixir_is_default_clause<'a>( - node: &Node<'a>, - code: &'a [u8], - ancestors: Ancestors<'a, '_>, -) -> bool { - use Elixir as E; - - let Some(pattern) = elixir_sole_unguarded_pattern(node) else { - return false; - }; - match pattern.kind_id().into() { - // Bare `_` catch-all — free everywhere except under an - // anonymous fn, whose free base path the head-clause skip - // already provides. - E::Identifier => { - pattern.utf8_text(code) == Some("_") - && ancestors - .parent(node) - .is_none_or(|parent| parent.kind_id() != E::AnonymousFunction as u16) - } - // Unguarded `true` — free only as `cond`'s designated - // default; the O(1) parent/grandparent checks run after the - // text compare so non-`true` booleans bail early. - E::Boolean => { - pattern.utf8_text(code) == Some("true") && elixir_is_cond_clause(node, code, ancestors) - } - _ => false, - } -} - -/// Returns `true` when `node` is the first `stab_clause` child of an -/// `anonymous_function` parent — the closure's head clause rather than a -/// pattern-dispatch branch. -/// -/// The grammar shape is `anonymous_function → fn stab_clause+ end`, so -/// the first child whose kind is `stab_clause` (skipping the `fn` token) -/// is the head. Multi-clause `fn`s thus skip only their first clause; -/// every other construct's clauses have a block parent and never match -/// here (issue #776). -fn elixir_is_anonymous_fn_head_clause<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - use Elixir as E; - - ancestors - .parent(node) - .filter(|parent| parent.kind_id() == E::AnonymousFunction as u16) - .and_then(|parent| { - parent - .children() - .find(|child| child.kind_id() == E::StabClause as u16) - }) - .is_some_and(|first| first.id() == node.id()) -} - -/// Whether `node` (a `stab_clause`) is one decision of the construct -/// that owns it: every clause of a `case` / `cond` / `with`'s `else` / -/// `receive` / `try` handler and every clause after a multi-clause -/// `fn`'s first, except the construct's free default clause -/// ([`elixir_is_default_clause`]). -/// -/// An anonymous fn's head clause is the closure's definition, not a -/// dispatch: the closure opens its own function space whose base path -/// already covers it, so counting it too would score a trivial -/// `fn x -> x end` as a decision (#776). -/// -/// The parent is allowlisted because the grammar puts a `stab_clause` -/// in one more position: a parenthesised `block`, which is how a -/// typespec spells a function type (`@spec f((any -> any)) :: list`). -/// That is type syntax and branches on nothing, the same reason -/// [`elixir_when_is_guard`] excludes a typespec's `when` (#1531). The -/// keyword form of a clause construct (`case(x, do: (1 -> :a; …))`) -/// holds its clauses in the same parenthesised `block`, so a `block` -/// counts when it is the value of a section pair -/// ([`elixir_is_section_pair`]). -/// -/// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode`, which -/// both score a clause construct per clause and must agree on which -/// clauses those are (grammar-dispatch §7, #1531). -pub(crate) fn elixir_clause_is_decision<'a>( - node: &Node<'a>, - code: &'a [u8], - ancestors: Ancestors<'a, '_>, -) -> bool { - use Elixir as E; - - let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); - let in_section = up - .next() - .is_some_and(|parent| match parent.kind_id().into() { - E::DoBlock - | E::ElseBlock - | E::AfterBlock - | E::RescueBlock - | E::CatchBlock - | E::AnonymousFunction => true, - E::Block => up - .next() - .is_some_and(|pair| elixir_is_section_pair(&pair, code, SECTION_KEYS)), - _ => false, - }); - in_section - && !elixir_is_anonymous_fn_head_clause(node, ancestors) - && !elixir_is_default_clause(node, code, ancestors) -} - -/// The keyword spellings of the clause sections a `do … end` block -/// spells as `do_block` / `else_block` / `after_block` / `rescue_block` -/// / `catch_block`. -const SECTION_KEYS: &[&str] = &["do:", "else:", "after:", "rescue:", "catch:"]; - -/// Whether `pair` is a keyword-form section (`do: (…)`) whose key is one -/// of `keys`. The key token's text carries its colon and the whitespace -/// after it, so it is trimmed before comparing. -fn elixir_is_section_pair(pair: &Node, code: &[u8], keys: &[&str]) -> bool { - pair.kind_id() == Elixir::Pair as u16 - && pair - .child_by_field_name("key") - .and_then(|key| key.utf8_text(code)) - .is_some_and(|key| keys.contains(&key.trim_end())) -} - -/// Whether `node` (a `stab_clause`) is one arm of a `cond`: it sits in -/// the `do` section of a `Call` spelling `cond`, either the block form's -/// `do_block` (whose parent is the call) or the keyword form's -/// `do: (…)` block (`block` → `pair` → `keywords` → `arguments` → call). -fn elixir_is_cond_clause<'a>( - node: &Node<'a>, - code: &'a [u8], - ancestors: Ancestors<'a, '_>, -) -> bool { - use Elixir as E; - - let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); - let call = match up.next().map(|parent| parent.kind_id().into()) { - Some(E::DoBlock) => up.next(), - Some(E::Block) => up - .next() - .filter(|pair| elixir_is_section_pair(pair, code, &["do:"])) - .and_then(|_| up.nth(2)), - _ => None, - }; - call.is_some_and(|call| { - crate::lang_helpers::elixir::elixir_call_keyword(&call, code) == Some("cond") - }) -} - // A `visibility_modifier` node counts as public unless it has a direct // `Zelf` child — the structural signature of `pub(self)` / `pub(in self)`, // which restrict visibility to the current module (semantically private, diff --git a/src/metrics/perl_keyword_key_tests.rs b/src/metrics/perl_keyword_key_tests.rs new file mode 100644 index 00000000..e16500af --- /dev/null +++ b/src/metrics/perl_keyword_key_tests.rs @@ -0,0 +1,644 @@ +//! An auto-quoted `and` or `not` key scores like any other key, and the +//! `and` and `not` operators still score as operators (#1539, #1541). +//! The same holds for a `-bareword` key and the file tests the grammar +//! mistakes it for (#1545). +//! +//! Perl quotes a bareword before `=>` and alone in a hash subscript, so +//! `$h{and}` and `(not => 1)` hold a string key and no operator. +//! tree-sitter-perl 1.1.2 lexes either bareword as the keyword anyway, +//! and every metric that keys on the token scored an operator, or a +//! decision, that is not there. `lang_helpers::perl::perl_and_is_operator` +//! and `perl_not_is_key` tell the two apart; these tests pin the metrics, +//! not the error tree the grammar builds around the key +//! (grammar-dispatch §6). +//! +//! Every row pairs its fixture with a twin that spells the same program +//! without the keyword token — an `or` key, which the grammar reads as a +//! plain identifier, or a `&&` / `!` operator — and the twin is the +//! oracle. The headline numbers are asserted outright as well, so a row +//! cannot pass by both spellings drifting together. + +use std::cell::Cell; + +use crate::test_support::check_func_space_only; +use crate::*; + +/// `[cyclomatic, abc conditions, cognitive]` sums of the root space. +/// The cyclomatic sum carries the unit's and the sub's base of 1 each. +#[cfg(feature = "perl")] +type Decisions = [u64; 3]; + +/// `[n1, N1, n2, N2]`. +#[cfg(feature = "perl")] +type Halstead = [u64; 4]; + +#[cfg(feature = "perl")] +fn measure(body: &str) -> (Decisions, Halstead) { + let ([cyclomatic, _, conditions, cognitive], halstead) = measure_with_branches(body); + ([cyclomatic, conditions, cognitive], halstead) +} + +/// `[cyclomatic, abc branches, abc conditions, cognitive]` sums of the +/// root space, and its Halstead counts. +#[cfg(feature = "perl")] +fn measure_with_branches(body: &str) -> ([u64; 4], Halstead) { + let out = Cell::new(([0; 4], [0; 4])); + check_func_space_only::( + &format!("sub f {{ {body} }}\n"), + "foo.pl", + &[ + Metric::Cyclomatic, + Metric::Abc, + Metric::Cognitive, + Metric::Halstead, + ], + |space| { + let m = &space.metrics; + out.set(( + [ + m.cyclomatic.cyclomatic_sum(), + m.abc.branches_sum(), + m.abc.conditions_sum(), + m.cognitive.cognitive_sum(), + ], + [ + m.halstead.unique_operators(), + m.halstead.total_operators(), + m.halstead.unique_operands(), + m.halstead.total_operands(), + ], + )); + }, + ); + out.get() +} + +/// Every valid spelling of an `and` key scores exactly as an `or` key +/// does, in all four metrics. Before #1539 each subscript scored one +/// extra cyclomatic decision and billed `and` as an operator, and each +/// `=>` key also scored an ABC condition and a cognitive sequence. +#[cfg(feature = "perl")] +#[test] +fn perl_and_key_scores_like_an_or_key() { + // `(or_twin, and_key, decisions)`. Three rows read the key's *value*: + // the ABC peel steps through an `and` key to it, as it does through a + // `not` key, so a negated value pays its condition. The last is a + // real subtraction before the key, whose `-` still bills. + let rows: [(&str, &str, Decisions); 12] = [ + ("my %h; return $h{or};", "my %h; return $h{and};", [2, 0, 0]), + ( + "my %h; return $h{ or };", + "my %h; return $h{ and };", + [2, 0, 0], + ), + ( + "my $r; return $r->{or};", + "my $r; return $r->{and};", + [2, 0, 0], + ), + ( + "my %h; if ($h{or}) { return 1; } return 0;", + "my %h; if ($h{and}) { return 1; } return 0;", + [3, 1, 1], + ), + ( + "my %h = (or => 1); return 1;", + "my %h = (and => 1); return 1;", + [2, 0, 0], + ), + ( + "my $h = { or => 1 }; return $h;", + "my $h = { and => 1 }; return $h;", + [2, 0, 0], + ), + ("f(x => 1, or => 2);", "f(x => 1, and => 2);", [2, 0, 0]), + ("f(or => !$a);", "f(and => !$a);", [2, 1, 0]), + ( + "my $y = $c ? 1 : (or => !$a);", + "my $y = $c ? 1 : (and => !$a);", + [3, 3, 1], + ), + ( + "if (or => $x > 1) { return 1; } return 0;", + "if (and => $x > 1) { return 1; } return 0;", + [3, 1, 1], + ), + ( + "my @a = (1 - or => 2);", + "my @a = (1 - and => 2);", + [2, 0, 0], + ), + // The infix `-` is the byte before the first key but signs + // neither: both keys are spelled `and`. + ( + "my @a = ($x - or => 2, or => 3);", + "my @a = ($x - and => 2, and => 3);", + [2, 0, 0], + ), + ]; + for (twin, key, decisions) in rows { + assert_eq!( + key.replace("and", "or"), + twin, + "`{key}` drifted from its twin" + ); + let want = measure(twin); + assert_eq!(want.0, decisions, "`{twin}` moved; re-derive the row"); + assert_eq!(measure(key), want, "`{key}` must score like `{twin}`"); + } +} + +/// A list holding an `and` key, assigned in a condition, is one +/// truthiness test: the slot evaluates to the list's last element, so +/// it pays one condition like any other value. +/// +/// This row guards cyclomatic, cognitive and Halstead, but not ABC. The +/// ABC count comes out at 1 whether or not the `and` gate in +/// `perl_is_logical_chain` holds. Without the gate, the slot takes the +/// key for a chain and scores 0, and the chain walk then pays the 1 the +/// slot dropped. `perl_and_key_scores_like_an_or_key` guards that gate. +#[cfg(feature = "perl")] +#[test] +fn perl_and_key_in_a_condition_slot_pays_the_slot() { + let twin = "my %h; if (%h = (or => 1)) { return 1; } return 0;"; + let key = "my %h; if (%h = (and => 1)) { return 1; } return 0;"; + let want = measure(twin); + assert_eq!(want.0, [3, 1, 1], "`{twin}` moved; re-derive the row"); + assert_eq!(measure(key), want, "`{key}` must score like `{twin}`"); +} + +/// A real low-precedence `and` keeps scoring as `&&` does — alone, +/// chained, after `not`, and inside the comparison and ternary shapes +/// the #1273 workaround climbs. Halstead is left out: `and` and `&&` +/// are distinct operators, and the ternary twin needs parentheses. +#[cfg(feature = "perl")] +#[test] +fn perl_and_operator_still_scores_like_ampamp() { + // `(ampamp_twin, and_operator, decisions)`. + let rows: [(&str, &str, Decisions); 5] = [ + ("return ($a && $b);", "return ($a and $b);", [3, 2, 1]), + ( + "return ($a && $b && $c);", + "return ($a and $b and $c);", + [4, 3, 1], + ), + ( + "if (!$a && $b) { return 1; } return 0;", + "if (not $a and $b) { return 1; } return 0;", + [4, 2, 2], + ), + ( + "return ($a > $b && $c);", + "return ($a > $b and $c);", + [3, 2, 1], + ), + ( + "return (($a > $b) && ($c ? 1 : 2));", + "return ($a > $b and $c ? 1 : 2);", + [4, 3, 2], + ), + ]; + for (twin, operator, decisions) in rows { + assert!(operator.contains(" and "), "`{operator}` lost its `and`"); + let want = measure(twin).0; + assert_eq!(want, decisions, "`{twin}` moved; re-derive the row"); + assert_eq!( + measure(operator).0, + want, + "`{operator}` must score like `{twin}`" + ); + } +} + +/// Every valid spelling of a `not` key scores exactly as an `or` key +/// does. Before #1541 each billed `not` as an operator where the twin +/// bills an operand, and a `=>` key outside a boolean slot also scored +/// an ABC condition, the peel reading it as `!` applied to its value. +/// The peel must still reach that value, which a negation there proves +/// boolean like any other. +#[cfg(feature = "perl")] +#[test] +fn perl_not_key_scores_like_an_or_key() { + // `(or_twin, not_key, decisions, halstead)`. + let rows: [(&str, &str, Decisions, Halstead); 13] = [ + ( + "my %h; return $h{or};", + "my %h; return $h{not};", + [2, 0, 0], + [6, 8, 4, 4], + ), + ( + "my %h; return $h{ or };", + "my %h; return $h{ not };", + [2, 0, 0], + [6, 8, 4, 4], + ), + ( + "my $r; return $r->{or};", + "my $r; return $r->{not};", + [2, 0, 0], + [7, 10, 3, 4], + ), + ( + "my %h = (or => 1); return 1;", + "my %h = (not => 1); return 1;", + [2, 0, 0], + [8, 9, 4, 5], + ), + ( + "my $r = { or => 1 }; return 1;", + "my $r = { not => 1 }; return 1;", + [2, 0, 0], + [8, 10, 4, 5], + ), + ( + "f(x => 1, or => 2);", + "f(x => 1, not => 2);", + [2, 0, 0], + [6, 7, 5, 6], + ), + ("f(or => 1);", "f(not => 1);", [2, 0, 0], [5, 5, 3, 4]), + ( + "return (or => 1);", + "return (not => 1);", + [2, 0, 0], + [6, 6, 3, 3], + ), + // The key's value is still a slot: the `!` proves it boolean. + ("f(or => !$a);", "f(not => !$a);", [2, 1, 0], [7, 7, 3, 4]), + ( + "my $y = $c ? 1 : (or => 2);", + "my $y = $c ? 1 : (not => 2);", + [3, 2, 1], + [10, 11, 6, 6], + ), + // A real subtraction before the key: the `-` is infix, no sign, + // and still bills. + ( + "my @a = ($x - or => 2);", + "my @a = ($x - not => 2);", + [2, 0, 0], + [9, 9, 5, 5], + ), + // A comment ending in `-` before the key signs nothing. + ( + "my %h = (\n # -----\n or => 1,\n or => 2,\n);", + "my %h = (\n # -----\n not => 1,\n not => 2,\n);", + [2, 0, 0], + [8, 10, 5, 6], + ), + // A `not` key after an earlier keyword key parses as `not` applied + // to a zero-width bareword the recovery invents. + ( + "f(and => 1, or => 2);", + "f(and => 1, not => 2);", + [2, 0, 0], + [6, 7, 5, 6], + ), + ]; + for (twin, key, decisions, halstead) in rows { + assert_eq!( + key.replace("not", "or"), + twin, + "`{key}` drifted from its twin" + ); + let want = measure(twin); + assert_eq!( + want, + (decisions, halstead), + "`{twin}` moved; re-derive the row" + ); + assert_eq!(measure(key), want, "`{key}` must score like `{twin}`"); + } +} + +/// A real `not` keeps scoring as `!` does, in all four metrics: alone, +/// in a condition, applied to a list, as a call argument the negation +/// proves boolean, ahead of a low-precedence `and`, and as the value of +/// a `not` key, where the key's recovery node sits one level up. `not` +/// and `!` are distinct operators, but each fixture spells only one of +/// them, so the Halstead counts agree too. +#[cfg(feature = "perl")] +#[test] +fn perl_not_operator_still_scores_like_bang() { + // `(bang_twin, not_operator, decisions, halstead)`. + let rows: [(&str, &str, Decisions, Halstead); 6] = [ + ("!$a;", "not $a;", [2, 0, 0], [5, 5, 2, 2]), + ( + "if (!$a) { return 1; } return 0;", + "if (not $a) { return 1; } return 0;", + [3, 1, 1], + [8, 11, 4, 4], + ), + ("!($a);", "not($a);", [2, 0, 0], [6, 6, 2, 2]), + ("f(!$a);", "f(not $a);", [2, 1, 0], [6, 6, 2, 3]), + ( + "return (!$a && $b);", + "return (not $a and $b);", + [3, 2, 1], + [8, 9, 3, 3], + ), + ( + "my %h = (or => !$a); return 1;", + "my %h = (not => not $a); return 1;", + [2, 0, 0], + [10, 11, 5, 5], + ), + ]; + for (twin, operator, decisions, halstead) in rows { + assert!(operator.contains("not"), "`{operator}` lost its `not`"); + let want = measure(twin); + assert_eq!( + want, + (decisions, halstead), + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure(operator), + want, + "`{operator}` must score like `{twin}`" + ); + } +} + +/// `(quoted_twin, dash_key, [cyclomatic, branches, conditions, +/// cognitive], halstead)` rows for +/// `perl_dash_key_scores_like_its_quoted_twin`. +#[cfg(feature = "perl")] +const DASH_KEY_ROWS: [(&str, &str, [u64; 4], Halstead); 20] = [ + ( + "my %h; return $h{'-foo'};", + "my %h; return $h{-foo};", + [2, 0, 0, 0], + [6, 8, 4, 4], + ), + // A comment in the subscript bills nothing, so this scores as the + // row above. Its `#` billed an operator until #1549 (7, 9). + ( + "my %h; return $h{'-foo' # why\n};", + "my %h; return $h{-foo # why\n};", + [2, 0, 0, 0], + [6, 8, 4, 4], + ), + ( + "my $r; return $r->{'-foo'};", + "my $r; return $r->{-foo};", + [2, 0, 0, 0], + [7, 10, 3, 4], + ), + ( + "my %h; return @h{'-foo'};", + "my %h; return @h{-foo};", + [2, 0, 0, 0], + [5, 7, 4, 4], + ), + ( + "my %h; if ($h{'-foo'}) { return 1; } return 0;", + "my %h; if ($h{-foo}) { return 1; } return 0;", + [3, 0, 1, 1], + [8, 13, 6, 6], + ), + ( + "my %h = ('-foo' => 1); return 1;", + "my %h = (-foo => 1); return 1;", + [2, 0, 0, 0], + [8, 9, 4, 5], + ), + ( + "my $r = { '-foo' => 1 }; return $r;", + "my $r = { -foo => 1 }; return $r;", + [2, 0, 0, 0], + [8, 11, 4, 5], + ), + ( + "f('-text' => 1);", + "f(-text => 1);", + [2, 1, 0, 0], + [5, 5, 3, 4], + ), + ( + "f('-text' => 1, ext => 2);", + "f(-text => 1, ext => 2);", + [2, 2, 0, 0], + [6, 7, 5, 6], + ), + ( + "my %h = ('-x' => 1); return 1;", + "my %h = (-x => 1); return 1;", + [2, 0, 0, 0], + [8, 9, 4, 5], + ), + ( + "my %h = ('-x' => 1); my %g = ('-x' => 2);", + "my %h = (-x => 1); my %g = (-x => 2);", + [2, 0, 0, 0], + [7, 12, 6, 7], + ), + // The swallowed value is still billed, call and all. + ( + "f('-x' => foo);", + "f(-x => foo);", + [2, 2, 0, 0], + [5, 5, 3, 4], + ), + ( + "my %h; return $h{'-not'};", + "my %h; return $h{-not};", + [2, 0, 0, 0], + [6, 8, 4, 4], + ), + ( + "my $r; return $r->{'-and'};", + "my $r; return $r->{-and};", + [2, 0, 0, 0], + [7, 10, 3, 4], + ), + ( + "my %h = ('-and' => 1); return 1;", + "my %h = (-and => 1); return 1;", + [2, 0, 0, 0], + [8, 9, 4, 5], + ), + ( + "my $r = { '-not' => 1 }; return $r;", + "my $r = { -not => 1 }; return $r;", + [2, 0, 0, 0], + [8, 11, 4, 5], + ), + ( + "f('-not' => 1, not => 2);", + "f(-not => 1, not => 2);", + [2, 1, 0, 0], + [6, 7, 5, 6], + ), + ( + "f('-and' => 1, and => 2);", + "f(-and => 1, and => 2);", + [2, 1, 0, 0], + [6, 7, 5, 6], + ), + // The ABC peel steps through the sign to the key and on to its + // value, so a negated value pays its condition, as its twin's does. + ( + "f('-and' => !$a);", + "f(-and => !$a);", + [2, 1, 1, 0], + [7, 7, 3, 4], + ), + ( + "f('-not' => !$a);", + "f(-not => !$a);", + [2, 1, 1, 0], + [7, 7, 3, 4], + ), +]; + +/// Each `-bareword` key below scores exactly as its quoted twin does, in +/// all four metrics (#1545). tree-sitter-perl reads `-foo` as the file +/// test `-f` on a bareword `oo`, `-x => 1` as the file test `-x` +/// swallowing the `=>`, and `-not` as a `-` applied to the keyword. +/// Before the fix each `-foo` billed the operand `oo` and an ABC branch +/// for calling it, and each `-not` / `-and` billed a `-` operator. A key +/// whose letter is no file-test letter (`-name`) is not among them: it +/// parses as a `-` applied to a bareword, which still bills the `-` and +/// an ABC branch for the word, as every bareword key does. +/// +/// The rows pairing a key with a second key equal to the misparsed +/// word (`ext`, `not`) or to itself (`-x` twice) pin the operand's +/// spelling, which the counts alone cannot see: billed as `ext`, `not` +/// or `-x => 1`, the two keys share or split one vocabulary entry +/// where their twins do not. +#[cfg(feature = "perl")] +#[test] +fn perl_dash_key_scores_like_its_quoted_twin() { + for (twin, key, decisions, halstead) in DASH_KEY_ROWS { + assert_eq!(twin.replace('\'', ""), key, "`{key}` drifted from its twin"); + let want = measure_with_branches(twin); + assert_eq!( + want, + (decisions, halstead), + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure_with_branches(key), + want, + "`{key}` must score like `{twin}`" + ); + } +} + +/// A `-and` / `-not` key keeps its sign when whitespace separates the +/// two: Perl keys `(- not => 1)` as `"-not"`, so it must not share a +/// vocabulary entry with the plain `not` key beside it. The twins' +/// values are `DASH_KEY_ROWS`' for the glued spelling. +#[cfg(feature = "perl")] +#[test] +fn perl_spaced_dash_key_keeps_its_sign() { + for (twin, key) in [ + ("f('-not' => 1, not => 2);", "f(- not => 1, not => 2);"), + ("f('-and' => 1, and => 2);", "f(- and => 1, and => 2);"), + // A comment between the sign and the key leaves it signed. + ( + "f('-not' => 1, not => 2);", + "f(- # c\n not => 1, not => 2);", + ), + ] { + let want = measure_with_branches(twin); + assert_eq!( + want, + ([2, 1, 0, 0], [6, 7, 5, 6]), + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure_with_branches(key), + want, + "`{key}` must score like `{twin}`" + ); + } +} + +/// Perl quotes a file-test `-X` before `=>` only across spaces and tabs. +/// Across a newline, or a comment, which runs to one, `-f` stays the +/// file test on `$_`: executed, `(-f # c⏎ => 7)` and `(-f⏎ => 7)` both +/// yield `(undef, 7)`, where `(-f \t => 7)` yields `("-f", 7)`. The +/// grammar recovers the same `ERROR` around the `=>` in every spelling, +/// and #1545 billed each as the key `-f`. Each row's twin is the program +/// Perl runs: the comma list for a file test, the quoted key otherwise. +/// A `-foo` key is quoted either way — `-foo` is the string `"-foo"` +/// whether or not the `=>` quotes it — so its commented and +/// line-broken spellings keep their key reading. +#[cfg(feature = "perl")] +#[test] +fn perl_file_test_key_stops_at_a_line_break() { + let comma = ([2, 0, 0, 0], [8, 9, 3, 4]); + let quoted = ([2, 0, 0, 0], [8, 9, 4, 5]); + let word = ([2, 1, 0, 0], [5, 5, 3, 4]); + for (twin, want, key) in [ + ( + "my @p = (-f, 7); return @p;", + comma, + "my @p = (-f # c\n=> 7); return @p;", + ), + ( + "my @p = (-f, 7); return @p;", + comma, + "my @p = (-f\n=> 7); return @p;", + ), + ( + "my @p = ('-f' => 7); return @p;", + quoted, + "my @p = (-f \t => 7); return @p;", + ), + ("f('-foo' => 1);", word, "f(-foo # c\n=> 1);"), + ("f('-foo' => 1);", word, "f(-foo\n=> 1);"), + ] { + assert_eq!( + measure_with_branches(twin), + want, + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure_with_branches(key), + want, + "`{key}` must score like `{twin}`" + ); + } +} + +/// A `-X` that is not a key keeps the reading it had before #1545: a +/// real file test, alone, in a condition, on the `_` stat cache, +/// stacked, and as a subscript's whole expression, where only a word +/// glued to the letter makes a key; a `-bareword` outside key position, +/// including a subscript it is only part of; +/// and a `-` applied to a real `not`. The file test itself is billed as +/// nothing, since the grammar emits its `-X` as a hidden token. Every +/// value is `main`'s, measured before the fix. +#[cfg(feature = "perl")] +#[test] +fn perl_file_test_outside_a_key_keeps_its_reading() { + let rows: [(&str, [u64; 4], Halstead); 11] = [ + ("my $z = -f $x;", [2, 0, 0, 0], [6, 7, 3, 3]), + ( + "if (-e $x) { return 1; } return 0;", + [3, 0, 1, 1], + [7, 10, 4, 4], + ), + ("my $z = -d _;", [2, 1, 0, 0], [6, 6, 3, 3]), + ("my $z = -f -w $x;", [2, 0, 0, 0], [6, 7, 3, 3]), + ("my %h; return $h{-f $x};", [2, 0, 0, 0], [6, 9, 4, 4]), + ("my %h; return $h{-f foo};", [2, 1, 0, 0], [6, 8, 4, 4]), + ("return -foo;", [2, 1, 0, 0], [4, 4, 2, 2]), + // Inside a subscript, but not the whole key. + ("my %h; return $h{$a . -foo};", [2, 1, 0, 0], [7, 10, 5, 5]), + ("f(-foo, 1);", [2, 2, 0, 0], [5, 5, 3, 4]), + ("return -not $x;", [2, 0, 0, 0], [7, 7, 2, 2]), + ("return $a - not $x;", [2, 0, 0, 0], [7, 8, 3, 3]), + ]; + for (body, decisions, halstead) in rows { + assert_eq!( + measure_with_branches(body), + (decisions, halstead), + "`{body}` changed its reading" + ); + } +} diff --git a/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__csharp_methods_arguments_with_conditions.snap b/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__csharp_methods_arguments_with_conditions.snap index ac6d3271..00a20ea8 100644 --- a/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__csharp_methods_arguments_with_conditions.snap +++ b/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__csharp_methods_arguments_with_conditions.snap @@ -5,16 +5,16 @@ expression: metric.abc { "assignments": 0, "branches": 2, - "conditions": 2, - "magnitude": 2.8284271247461903, + "conditions": 3, + "magnitude": 3.605551275463989, "value": 0.0, "assignments_average": 0.0, "branches_average": 0.5, - "conditions_average": 0.5, + "conditions_average": 0.75, "assignments_min": 0, "assignments_max": 0, "branches_min": 0, "branches_max": 2, "conditions_min": 0, - "conditions_max": 2 + "conditions_max": 3 } diff --git a/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__perl_short_circuit_not_counted_directly_ternary_counts.snap b/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__perl_short_circuit_not_counted_directly_ternary_counts.snap index d4816ee7..d28af22b 100644 --- a/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__perl_short_circuit_not_counted_directly_ternary_counts.snap +++ b/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__perl_short_circuit_not_counted_directly_ternary_counts.snap @@ -5,16 +5,16 @@ expression: metric.abc { "assignments": 7, "branches": 0, - "conditions": 14, - "magnitude": 15.652475842498529, + "conditions": 12, + "magnitude": 13.892443989449804, "value": 0.0, "assignments_average": 3.5, "branches_average": 0.0, - "conditions_average": 7.0, + "conditions_average": 6.0, "assignments_min": 0, "assignments_max": 7, "branches_min": 0, "branches_max": 0, "conditions_min": 0, - "conditions_max": 14 + "conditions_max": 12 } diff --git a/src/ops.rs b/src/ops.rs index b3583772..137982c5 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -461,7 +461,7 @@ mod tests { "if True: a = 1 + 2", "foo.py", - &mut ["if", "=", "+"], + &mut ["if", ":", "=", "+"], &mut ["True", "a", "1", "2"], ); } @@ -495,6 +495,20 @@ mod tests { ); } + /// The fat comma is a named node, so the generic fallback rendered + /// its kind name, `fat_comma`; it lists as the `=>` written. + #[cfg(feature = "perl")] + #[test] + fn perl_fat_comma_renders_as_written() { + check_ops( + LANG::Perl, + "my %h = (a => 1);", + "foo.pl", + &mut ["()", ";", "=", "=>", "my"], + &mut ["%h", "1", "a"], + ); + } + #[cfg(feature = "python")] #[test] fn python_function_ops() { @@ -507,11 +521,27 @@ mod tests { b = 2 + a c = 3 + 3", "foo.py", - &mut ["def", "=", "+"], + &mut ["def", "()", ":", "=", "+"], &mut ["foo", "bar", "toto", "a", "b", "c", "1", "2", "3"], ); } + /// #1486: Python bills a call's parentheses the way the JS family + /// does — one `()` entry for the pair, rendered as the pair. Before + /// it, Python reported no operator here at all. + #[cfg(all(feature = "javascript", feature = "python"))] + #[test] + fn python_call_parens_pair_like_javascript() { + check_ops(LANG::Python, "f(x)", "foo.py", &mut ["()"], &mut ["f", "x"]); + check_ops( + LANG::Javascript, + "f(x)", + "foo.js", + &mut ["()"], + &mut ["f", "x"], + ); + } + #[cfg(feature = "cpp")] #[test] fn cpp_ops() { @@ -562,7 +592,7 @@ mod tests { LANG::Rust, "let: usize a = 5; let b: f32 = 7.0; let c: i32 = 3;", "foo.rs", - &mut ["let", "usize", "=", ";", "f32", "i32"], + &mut ["let", ":", "usize", "=", ";", "f32", "i32"], &mut ["a", "b", "c", "5", "7.0", "3"], ); } @@ -873,11 +903,13 @@ mod tests { }", "foo.java", &mut [ - "{}", "void", "()", "[]", ",", ".", ";", "int", "=", "+", "/", + "class", "{}", "void", "()", "[]", ",", ".", ";", "int", "=", "+", "/", ], &mut [ "Main", "main", + // the parameter's type name (#1560) + "string", "args", "a", "b", @@ -910,8 +942,9 @@ mod tests { }", "foo.java", // All 8 primitive-type keywords must appear as distinct operators. - // true/false appear as operands. + // true/false appear as operands; `class` is billed since #1552. &mut [ + "class", "{}", ";", "=", @@ -1446,4 +1479,304 @@ mod tests { ], ); } + + /// #1559: a generic list's angle brackets are one `<>` bracket pair, + /// billed on the opener as `()` / `[]` / `{}` are, with the closer + /// billing nothing. Every fixture also spells a comparison `<` and + /// `>`, which must stay their own vocabulary entries: the grammars + /// give the list brackets the comparison operators' kind ids, so the + /// vocabulary is what shows the two apart and `N1` is what shows the + /// closer is not billed. + #[cfg(any( + feature = "cpp", + feature = "csharp", + feature = "groovy", + feature = "java", + feature = "kotlin", + feature = "mozcpp", + feature = "objc", + feature = "rust", + feature = "typescript", + ))] + fn check_generic_pair(lang: LANG, file: &str, source: &str, operators: &[&str], n1: [u64; 2]) { + let ast = crate::test_support::parse_named(lang, file, source); + let ops = ast.ops().expect("ops walk must yield a top-level Ops"); + let mut expected = operators.to_vec(); + expected.sort_unstable(); + assert_eq!(ops.operators, expected, "{lang:?} vocabulary"); + + let space = ast + .metrics(crate::MetricsOptions::default().with_only(&[crate::Metric::Halstead])) + .expect("metrics walk"); + let halstead = &space.metrics.halstead; + assert_eq!( + [halstead.unique_operators(), halstead.total_operators()], + n1, + "{lang:?} [n1, N1]" + ); + } + + // expected: `<>` three times (one parameter list, two nested + // argument lists), never `>`; `<` and `>` once each, the comparisons. + #[cfg(feature = "rust")] + #[test] + fn rust_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Rust, + "foo.rs", + "fn f(v: Vec>) -> bool { a < b && c > d }\n", + &["fn", "<>", "()", ":", "->", "bool", "{}", "<", "&&", ">"], + [10, 12], + ); + // A turbofish and a higher-ranked `for<'a>` (`for_lifetimes`) + // are lists too, `<>` twice (`fn` is twice as well); the `>>` shift + // is its own operator. + check_generic_pair( + LANG::Rust, + "foo.rs", + "fn g(f: for<'a> fn(&'a u8)) { h::(x >> 1); }\n", + &[ + "fn", "()", ":", "for", "<>", "&", "u8", "{}", "::", ">>", ";", + ], + [11, 16], + ); + // A precise-capturing `use` (`use_bounds`) is a list as well. + check_generic_pair( + LANG::Rust, + "foo.rs", + "fn u(x: T) -> impl Sized + use { x }\n", + &["fn", "<>", "()", ":", "->", "impl", "+", "use", "{}"], + [9, 10], + ); + } + + // expected: `<>` three times, `;` and `{}` twice each. + #[cfg(feature = "java")] + #[test] + fn java_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Java, + "foo.java", + "class A { List> v; boolean m() { return a < b && c > d; } }\n", + &[ + "class", + "<>", + ";", + "{}", + "boolean_type", + "()", + "return", + "<", + "&&", + ">", + ], + [10, 14], + ); + } + + // expected: as Java, with `bool` for `boolean_type`. + #[cfg(feature = "csharp")] + #[test] + fn csharp_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Csharp, + "foo.cs", + "class A { List> v; bool M() { return a < b && c > d; } }\n", + &[ + "class", "<>", ";", "{}", "bool", "()", "return", "<", "&&", ">", + ], + [10, 14], + ); + } + + // expected: `<>` three times, `:` twice. TSX shares the getter macro + // and the same arm, so it is checked on the same input. + #[cfg(feature = "typescript")] + #[test] + fn typescript_generic_angles_are_one_pair_1559() { + const SOURCE: &str = + "function f(v: Array>): boolean { return a < b && c > d; }\n"; + const OPERATORS: &[&str] = &[ + "function", "<>", "()", ":", "boolean", "{}", "return", "<", "&&", ">", ";", + ]; + check_generic_pair(LANG::Typescript, "foo.ts", SOURCE, OPERATORS, [11, 14]); + check_generic_pair(LANG::Tsx, "foo.tsx", SOURCE, OPERATORS, [11, 14]); + } + + // expected: `<>` three times (the `template` parameter list and two + // nested argument lists), `::` twice. The closer of the outer list + // is spelled `>>` and still parses as two `>` tokens. + #[cfg(feature = "cpp")] + #[test] + fn cpp_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Cpp, + "foo.cpp", + "template bool f(std::vector> v) { return a < b && c > d; }\n", + &[ + "template", "<>", "class", "bool", "()", "::", "{}", "return", "<", "&&", ">", ";", + ], + [12, 15], + ); + } + + // expected: the Cpp row, through the Mozcpp grammar, which owns no + // file extension and so has no corpus coverage of its own. + #[cfg(feature = "mozcpp")] + #[test] + fn mozcpp_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Mozcpp, + "foo.cpp", + "template bool f(std::vector> v) { return a < b && c > d; }\n", + &[ + "template", "<>", "class", "bool", "()", "::", "{}", "return", "<", "&&", ">", ";", + ], + [12, 15], + ); + } + + // expected: `<>` three times, `:` twice. + #[cfg(feature = "kotlin")] + #[test] + fn kotlin_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Kotlin, + "foo.kt", + "fun f(v: List>): Boolean { return a < b && c > d }\n", + &["fun", "<>", "()", ":", "{}", "return", "<", "&&", ">"], + [9, 12], + ); + } + + // expected: `<>` four times — Groovy spells a method's parameter + // list as its own `method_type_parameters` node. + #[cfg(feature = "groovy")] + #[test] + fn groovy_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Groovy, + "foo.groovy", + "class A { List> v; def boolean m() { return a < b && c > d } }\n", + &[ + "class", "<>", ";", "{}", "def", "()", "return", "<", "&&", ">", + ], + [10, 14], + ); + } + + // expected: `<>` five times, once per list — `parameterized_arguments` + // (`: NSObject `), `protocol_reference_list` + // (`id`), two nested `generic_specifier`s, and the + // `argument_list` type form `g(NSArray)` — and never `>`. + // `*` four times, `()` and `||` twice; `@end` and `id` bill nothing. + // Without the pair, `<` and `>` would each bill six times: [10, 25]. + // expected: the spaced twin `g(a < b, c > d)`, two comparisons. + // Unspaced, tree-sitter-objc reads the arguments as an + // `argument_list`'s `Type<…>` form with an `ERROR` after the `>`, and + // they must keep billing `<` and `>`; the clean `g(NSArray)` + // stays one pair. + #[cfg(feature = "objc")] + #[test] + fn objc_misparsed_comparisons_stay_comparisons_1559() { + const COMPARISONS: &[&str] = &["()", ",", ";", "<", ">", "int", "return", "void", "{}"]; + check_generic_pair( + LANG::Objc, + "foo.m", + "int f(void) { return g(a < b, c > d); }\n", + COMPARISONS, + [9, 10], + ); + check_generic_pair( + LANG::Objc, + "foo.m", + "int f(void) { return g(ad); }\n", + COMPARISONS, + [9, 10], + ); + check_generic_pair( + LANG::Objc, + "foo.m", + "int f(void) { return g(NSArray); }\n", + &["()", "*", ";", "<>", "int", "return", "void", "{}"], + [8, 9], + ); + } + + // expected: the reading before #1559, which billed the two + // comparisons as `<` and `>`. By the C# specification the identifier + // after the `>` makes `F(a < b, c > d)` two arguments, though the + // grammar parses a declaration typed `a`. An `out` declaration + // is a real generic and stays one pair. + #[cfg(feature = "csharp")] + #[test] + fn csharp_misparsed_comparisons_stay_comparisons_1559() { + check_generic_pair( + LANG::Csharp, + "foo.cs", + "class A { void M() { F(a < b, c > d); } }\n", + &["()", ",", ";", "<", ">", "class", "void", "{}"], + [8, 10], + ); + // A member access before the `<` wraps the generic name in a + // `qualified_name`; the comparisons are the same. + check_generic_pair( + LANG::Csharp, + "foo.cs", + "class A { void M() { F(x.a < b, c > d); } }\n", + &["()", ",", ".", ";", "<", ">", "class", "void", "{}"], + [9, 11], + ); + check_generic_pair( + LANG::Csharp, + "foo.cs", + "class A { void M() { F(out List x); } }\n", + &["()", ";", "<>", "class", "int", "out", "void", "{}"], + [8, 10], + ); + // A deconstruction's declarations are `argument`s of a + // `tuple_expression`, not of an argument list: a real generic. + check_generic_pair( + LANG::Csharp, + "foo.cs", + "class A { void M() { (List p, int q) = (1, 2); } }\n", + &["()", ",", ";", "<>", "=", "class", "int", "void", "{}"], + [9, 14], + ); + // A using alias's generic name sits two levels under the root, too + // shallow for the argument-list shape. + check_generic_pair( + LANG::Csharp, + "foo.cs", + "using X = List;\n", + &[";", "<>", "=", "int", "using"], + [5, 5], + ); + } + + #[cfg(feature = "objc")] + #[test] + fn objc_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Objc, + "foo.m", + "@interface A : NSObject \n@end\n\ + id f(NSArray *> *v) \ + { return g(NSArray) || a < b || c > d; }\n", + &[ + "@interface", + ":", + "<>", + "()", + "*", + "{}", + "return", + "||", + "<", + ">", + ";", + ], + [11, 20], + ); + } } diff --git a/src/spaces_tests.rs b/src/spaces_tests.rs index 51cbbc3d..6ff65d68 100644 --- a/src/spaces_tests.rs +++ b/src/spaces_tests.rs @@ -91,6 +91,132 @@ fn cpp_function_definition_is_classified_as_function() { ); } +/// #1555 fixture: five bodiless specifiers (two forward declarations, +/// an elaborated variable type, and two elaborated parameter types) +/// beside four bodied definitions, including the `final` / base-clause, +/// template and anonymous forms whose `body` follows optional preambles. +#[cfg(any(feature = "cpp", feature = "mozcpp"))] +const CPP_BODILESS_SPECIFIERS: &str = "struct S; +struct S *p; +class Fwd; +void f(struct S *q); +void g(class Fwd *r) {} +struct T {} t; +class C final : public Fwd { int x; }; +template struct X {}; +struct {} anon; +"; + +/// #1555: only a struct / class specifier with a body opens a space. The +/// bodiless ones used to open one each, at unit level even when they sat +/// in a parameter list. +#[cfg(any(feature = "cpp", feature = "mozcpp"))] +fn assert_only_bodied_specifiers_open_spaces() { + check_func_space::(CPP_BODILESS_SPECIFIERS, "fwd.cpp", |root| { + let top: Vec<(SpaceKind, Option<&str>, usize)> = root + .spaces + .iter() + .map(|s| (s.kind, s.name.as_deref(), s.start_line)) + .collect(); + assert_eq!( + top, + vec![ + (SpaceKind::Function, Some("g"), 5), + (SpaceKind::Struct, Some("T"), 6), + (SpaceKind::Class, Some("C"), 7), + (SpaceKind::Struct, Some("X"), 8), + (SpaceKind::Struct, None, 9), + ], + "top-level spaces (kind, name, line); got {top:?}" + ); + assert!( + root.spaces.iter().all(|s| s.spaces.is_empty()), + "no space nests below these: `g`'s `class Fwd` parameter type opens none" + ); + }); +} + +#[cfg(feature = "cpp")] +#[test] +fn cpp_bodiless_specifiers_open_no_space_1555() { + assert_only_bodied_specifiers_open_spaces::(); +} + +#[cfg(feature = "mozcpp")] +#[test] +fn mozcpp_bodiless_specifiers_open_no_space_1555() { + assert_only_bodied_specifiers_open_spaces::(); +} + +/// #1555: `promotes_to_func_space_with_code` and +/// `get_space_kind_with_code` — the spellings the walker calls — agree on +/// every struct / class specifier node, checked node by node because the +/// space tree alone cannot see the getter half: it only labels spaces the +/// checker already opened. Today both forward to `is_func_space` / +/// `get_space_kind`, so only a `_with_code` override could split them +/// (grammar-dispatch §7). The oracle is independent of the `body` field: +/// a specifier defines a type exactly when its text has a `{`. +#[cfg(any(feature = "cpp", feature = "mozcpp"))] +fn assert_specifier_predicates_agree(struct_id: u16, class_id: u16) { + use crate::checker::Checker; + use crate::getter::Getter; + + let parser = T::new( + CPP_BODILESS_SPECIFIERS.as_bytes().to_vec(), + &std::path::PathBuf::from("fwd.cpp"), + None, + ); + let code = CPP_BODILESS_SPECIFIERS.as_bytes(); + let (mut bodied, mut bodiless) = (0, 0); + for node in parser.root().preorder() { + let id = node.kind_id(); + let kind = if id == struct_id { + SpaceKind::Struct + } else if id == class_id { + SpaceKind::Class + } else { + continue; + }; + let has_body = code[node.start_byte()..node.end_byte()].contains(&b'{'); + if has_body { + bodied += 1; + } else { + bodiless += 1; + } + assert_eq!( + T::Checker::promotes_to_func_space_with_code(&node, code, Ancestors::unknown()), + has_body, + "{node:?}" + ); + assert_eq!( + T::Getter::get_space_kind_with_code(&node, code, Ancestors::unknown()), + if has_body { kind } else { SpaceKind::Unknown }, + "{node:?}" + ); + } + assert_eq!((bodied, bodiless), (4, 5), "fixture specifier census"); +} + +#[cfg(feature = "cpp")] +#[test] +fn cpp_specifier_space_predicates_agree_1555() { + use crate::Cpp; + assert_specifier_predicates_agree::( + Cpp::StructSpecifier as u16, + Cpp::ClassSpecifier as u16, + ); +} + +#[cfg(feature = "mozcpp")] +#[test] +fn mozcpp_specifier_space_predicates_agree_1555() { + use crate::Mozcpp; + assert_specifier_predicates_agree::( + Mozcpp::StructSpecifier as u16, + Mozcpp::ClassSpecifier as u16, + ); +} + #[cfg(feature = "cpp")] #[test] fn cpp_scope_resolution_operator() { @@ -1886,8 +2012,6 @@ mod metric_selection_parity { // multi-clause `if` (cognitive, cyclomatic, abc, halstead, tokens), // an early `return` (nexits), parameters (nargs), and a comment // (loc). It is the fixture the non-vacuity assertion below leans on. - // test-lang-gates: hand-written(rust) — a source fixture for - // this grammar, and its text is a string #[cfg(feature = "rust")] const RUST: &str = "\ pub struct Counter { @@ -1916,8 +2040,6 @@ fn choose(a: u32, b: u32) -> u32 { } "; - // test-lang-gates: hand-written(java) — a source fixture for - // this grammar, and its text is a string #[cfg(feature = "java")] const JAVA: &str = "\ public class Shape { @@ -1933,8 +2055,6 @@ public class Shape { } "; - // test-lang-gates: hand-written(python) — a source fixture for - // this grammar, and its text is a string #[cfg(feature = "python")] const PYTHON: &str = "\ class Bag: diff --git a/tests/api/ast_seam_test.rs b/tests/api/ast_seam_test.rs index 02ef30ca..3cd68aa7 100644 --- a/tests/api/ast_seam_test.rs +++ b/tests/api/ast_seam_test.rs @@ -763,14 +763,10 @@ fn preprocess_harvest_feeds_the_macro_masking_pass() { /// failing (`.claude/rules/testing.md`, "Perturb the fixture as well as /// the production line"). Inside, the `proc` is load-bearing for the /// expected sequence. -// test-lang-gates: hand-written(tcl) — a source fixture for this -// grammar, and its text is a string #[cfg(feature = "tcl")] const TCL_SCRIPT_AND_LITERALS: &str = "proc p {x} { puts \"q\" }\nlappend l {a b}\n"; /// The iRules twin, which already had the quoted word inside the body. -// test-lang-gates: hand-written(irules) — a source fixture for this -// grammar, and its text is a string #[cfg(feature = "irules")] const IRULES_SCRIPT_AND_LITERALS: &str = "when HTTP_REQUEST { log local0. \"hi\" }\nlappend l {x y}\n"; diff --git a/tests/corpus/csharp_test.rs b/tests/corpus/csharp_test.rs index 177f7b1f..68a3fb86 100644 --- a/tests/corpus/csharp_test.rs +++ b/tests/corpus/csharp_test.rs @@ -21,5 +21,5 @@ fn test_csharp() { .join("repositories") .join("big-code-analysis-output"); - compare_rca_output_with_files_under(&source_root, "csharp", &["*.cs"], &[], 6); + compare_rca_output_with_files_under(&source_root, "csharp", &["*.cs"], &[], 7); } diff --git a/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap b/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap index ab931ab1..ca5892f2 100644 --- a/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap +++ b/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap @@ -3,8 +3,8 @@ source: tests/output_formats/csv_test.rs expression: out --- path,space_name,space_kind,start_line,end_line,cognitive.sum,cognitive.average,cognitive.min,cognitive.max,cyclomatic.sum,cyclomatic.average,cyclomatic.min,cyclomatic.max,cyclomatic.modified.sum,cyclomatic.modified.average,cyclomatic.modified.min,cyclomatic.modified.max,halstead.unique_operators,halstead.total_operators,halstead.unique_operands,halstead.total_operands,halstead.length,halstead.estimated_program_length,halstead.purity_ratio,halstead.vocabulary,halstead.volume,halstead.difficulty,halstead.level,halstead.effort,halstead.time,halstead.bugs,loc.sloc,loc.ploc,loc.lloc,loc.cloc,loc.blank,loc.sloc_average,loc.ploc_average,loc.lloc_average,loc.cloc_average,loc.blank_average,loc.sloc_min,loc.sloc_max,loc.cloc_min,loc.cloc_max,loc.ploc_min,loc.ploc_max,loc.lloc_min,loc.lloc_max,loc.blank_min,loc.blank_max,nom.functions,nom.closures,nom.functions_average,nom.closures_average,nom.total,nom.average,nom.functions_min,nom.functions_max,nom.closures_min,nom.closures_max,nargs.function_args,nargs.closure_args,nargs.function_args_average,nargs.closure_args_average,nargs.total,nargs.average,nargs.function_args_min,nargs.function_args_max,nargs.closure_args_min,nargs.closure_args_max,nexits.sum,nexits.average,nexits.min,nexits.max,tokens.sum,tokens.average,tokens.min,tokens.max,abc.assignments,abc.branches,abc.conditions,abc.magnitude,abc.assignments_average,abc.branches_average,abc.conditions_average,abc.assignments_min,abc.assignments_max,abc.branches_min,abc.branches_max,abc.conditions_min,abc.conditions_max,wmc.class_wmc_sum,wmc.interface_wmc_sum,wmc.total,npm.class_npm_sum,npm.interface_npm_sum,npm.class_methods,npm.interface_methods,npm.class_coa,npm.interface_coa,npm.total,npm.total_methods,npm.coa,npa.class_npa_sum,npa.interface_npa_sum,npa.class_attributes,npa.interface_attributes,npa.class_cda,npa.interface_cda,npa.total,npa.total_attributes,npa.cda,mi.original,mi.sei,mi.visual_studio -widget.cc,widget.cc,unit,1,10,0,0,0,0,5,2.5,1,1,5,2.5,1,1,8,18,6,9,27,39.50977500432694,1.4633250001602571,14,102.79858289555531,6,0.16666666666666666,616.7914973733318,34.266194298518435,0.024152995332196794,10,9,2,0,1,2,1.8,0.4,0,0.2,1,10,0,0,1,9,1,2,0,1,2,0,0.4,0,2,0.4,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,7.6,0,12,1,0,0,1,0.2,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,108.457709340607,81.2796467168799,63.42556101789883 -widget.cc,ns,namespace,2,10,0,0,0,0,4,2,1,1,4,2,1,1,8,18,6,9,27,39.50977500432694,1.4633250001602571,14,102.79858289555531,6,0.16666666666666666,616.7914973733318,34.266194298518435,0.024152995332196794,9,9,2,0,0,2.25,2.25,0.5,0,0,1,9,0,0,1,9,1,2,0,0,2,0,0.5,0,2,0.5,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,9.5,5,12,1,0,0,1,0.25,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,110.3945496942638,83.97209683068972,64.55821619547591 -widget.cc,Widget,class,3,9,0,0,0,0,3,1.5,1,1,3,1.5,1,1,8,16,5,8,24,35.60964047443681,1.4837350197682004,13,88.81055323538621,6.4,0.15625,568.3875407064718,31.57708559480399,0.022872226594196698,7,7,2,0,0,2.3333333333333335,2.3333333333333335,0.6666666666666666,0,0,1,7,0,0,1,7,1,2,0,0,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,33,11,10,12,1,0,0,1,0.3333333333333333,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,115.45642705977977,91.17302234410326,67.51837839753202 +widget.cc,widget.cc,unit,1,10,0,0,0,0,5,2.5,1,1,5,2.5,1,1,10,20,6,9,29,48.72905595320056,1.680312274248295,16,116,7.5,0.13333333333333333,870,48.333333333333336,0.030377934896425568,10,9,2,0,1,2,1.8,0.4,0,0.2,1,10,0,0,1,9,1,2,0,1,2,0,0.4,0,2,0.4,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,7.6,0,12,1,0,0,1,0.2,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,107.82945249974337,80.37326368816134,63.05815935657507 +widget.cc,ns,namespace,2,10,0,0,0,0,4,2,1,1,4,2,1,1,10,20,6,9,29,48.72905595320056,1.680312274248295,16,116,7.5,0.13333333333333333,870,48.333333333333336,0.030377934896425568,9,9,2,0,0,2.25,2.25,0.5,0,0,1,9,0,0,1,9,1,2,0,0,2,0,0.5,0,2,0.5,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,9.5,5,12,1,0,0,1,0.25,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,109.76629285340017,83.06571380197117,64.19081453415215 +widget.cc,Widget,class,3,9,0,0,0,0,3,1.5,1,1,3,1.5,1,1,9,17,5,8,25,40.13896548741762,1.6055586194967049,14,95.1838730514401,7.2,0.1388888888888889,685.3238859703688,38.07354922057604,0.02591051177740593,7,7,2,0,0,2.3333333333333335,2.3333333333333335,0.6666666666666666,0,0,1,7,0,0,1,7,1,2,0,0,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,33,11,10,12,1,0,0,1,0.3333333333333333,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,115.09604084632235,90.65309494114345,67.3076262259195 widget.cc,value,function,5,5,0,0,0,0,1,1,1,1,1,1,1,1,5,5,2,2,7,13.60964047443681,1.944234353490973,7,19.651484454403228,2.5,0.4,49.12871113600807,2.7293728408893374,0.004471319041702271,1,1,1,0,0,1,1,1,0,0,1,1,0,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,10,10,10,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,155.2836050228807,148.4278547652622,90.80912574437467 widget.cc,set,function,6,6,0,0,0,0,1,1,1,1,1,1,1,1,6,6,3,4,10,20.264662506490403,2.0264662506490403,9,31.69925001442312,4,0.25,126.79700005769249,7.044277780982916,0.008413009854528818,1,1,1,0,0,1,1,1,0,0,1,1,0,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,12,12,12,12,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,152.79727628688778,144.8408406278255,89.35513233151332 diff --git a/tests/output_formats/snapshots/output_formats__csv_test__csv_python_greeter.snap b/tests/output_formats/snapshots/output_formats__csv_test__csv_python_greeter.snap index 264489d1..9ea8f483 100644 --- a/tests/output_formats/snapshots/output_formats__csv_test__csv_python_greeter.snap +++ b/tests/output_formats/snapshots/output_formats__csv_test__csv_python_greeter.snap @@ -3,7 +3,7 @@ source: tests/output_formats/csv_test.rs expression: out --- path,space_name,space_kind,start_line,end_line,cognitive.sum,cognitive.average,cognitive.min,cognitive.max,cyclomatic.sum,cyclomatic.average,cyclomatic.min,cyclomatic.max,cyclomatic.modified.sum,cyclomatic.modified.average,cyclomatic.modified.min,cyclomatic.modified.max,halstead.unique_operators,halstead.total_operators,halstead.unique_operands,halstead.total_operands,halstead.length,halstead.estimated_program_length,halstead.purity_ratio,halstead.vocabulary,halstead.volume,halstead.difficulty,halstead.level,halstead.effort,halstead.time,halstead.bugs,loc.sloc,loc.ploc,loc.lloc,loc.cloc,loc.blank,loc.sloc_average,loc.ploc_average,loc.lloc_average,loc.cloc_average,loc.blank_average,loc.sloc_min,loc.sloc_max,loc.cloc_min,loc.cloc_max,loc.ploc_min,loc.ploc_max,loc.lloc_min,loc.lloc_max,loc.blank_min,loc.blank_max,nom.functions,nom.closures,nom.functions_average,nom.closures_average,nom.total,nom.average,nom.functions_min,nom.functions_max,nom.closures_min,nom.closures_max,nargs.function_args,nargs.closure_args,nargs.function_args_average,nargs.closure_args_average,nargs.total,nargs.average,nargs.function_args_min,nargs.function_args_max,nargs.closure_args_min,nargs.closure_args_max,nexits.sum,nexits.average,nexits.min,nexits.max,tokens.sum,tokens.average,tokens.min,tokens.max,abc.assignments,abc.branches,abc.conditions,abc.magnitude,abc.assignments_average,abc.branches_average,abc.conditions_average,abc.assignments_min,abc.assignments_max,abc.branches_min,abc.branches_max,abc.conditions_min,abc.conditions_max,wmc.class_wmc_sum,wmc.interface_wmc_sum,wmc.total,npm.class_npm_sum,npm.interface_npm_sum,npm.class_methods,npm.interface_methods,npm.class_coa,npm.interface_coa,npm.total,npm.total_methods,npm.coa,npa.class_npa_sum,npa.interface_npa_sum,npa.class_attributes,npa.interface_attributes,npa.class_cda,npa.interface_cda,npa.total,npa.total_attributes,npa.cda,mi.original,mi.sei,mi.visual_studio -greeter.py,greeter.py,unit,1,9,1,0.5,0,1,5,2.5,1,2,5,2.5,1,2,6,10,6,14,24,31.019550008653873,1.292481250360578,12,86.03910001730775,7,0.14285714285714285,602.2737001211542,33.45965000673079,0.02377249024853662,9,7,4,0,2,2.25,1.75,1,0,0.5,2,9,0,0,2,7,1,4,0,2,2,0,0.5,0,2,0.5,0,1,0,0,3,0,1.5,0,3,1.5,0,2,0,0,2,1,0,2,41,10.25,0,25,1,0,1,1.4142135623730951,0.25,0,0.25,0,1,0,0,0,1,3,0,3,2,0,2,0,1,0,2,2,1,1,0,1,0,1,0,1,1,1,111.08999225681957,85.0772282263165,64.96490775252606 -greeter.py,Greeter,class,2,9,1,0.5,0,1,4,2,1,2,4,2,1,2,6,10,6,14,24,31.019550008653873,1.292481250360578,12,86.03910001730775,7,0.14285714285714285,602.2737001211542,33.45965000673079,0.02377249024853662,8,7,4,0,1,2.6666666666666665,2.3333333333333335,1.3333333333333333,0,0.3333333333333333,2,8,0,0,2,7,1,4,0,1,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,3,0,1.5,0,3,1.5,0,2,0,0,2,1,0,2,41,13.666666666666666,3,25,1,0,1,1.4142135623730951,0.3333333333333333,0,0.3333333333333333,0,1,0,0,0,1,3,0,3,2,0,2,0,1,0,2,2,1,1,0,1,0,1,0,1,1,1,113.22807743445301,88.06001324968199,66.21524996166842 -greeter.py,__init__,function,3,4,0,0,0,0,1,1,1,1,1,1,1,1,4,4,3,6,10,12.754887502163468,1.2754887502163468,7,28.07354922057604,4,0.25,112.29419688230416,6.2385664934613425,0.007758604623869012,2,2,1,0,0,2,2,1,0,0,2,2,0,0,2,2,1,1,0,0,1,0,1,0,1,1,1,1,0,0,2,0,2,0,2,2,2,2,0,0,0,0,0,0,13,13,13,13,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,142.19991098932817,129.5520742665475,83.15784268381765 -greeter.py,greet,function,6,9,1,1,1,1,2,2,2,2,2,2,2,2,4,6,4,7,13,16,1.2307692307692308,8,39,3.5,0.2857142857142857,136.5,7.583333333333333,0.008836911647049637,4,4,3,0,0,4,4,3,0,0,4,4,0,0,4,4,3,3,0,0,1,0,1,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,2,2,2,2,25,25,25,25,0,0,1,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,129.0315107899836,110.65590846191628,75.45702385379158 +greeter.py,greeter.py,unit,1,9,1,0.5,0,1,5,2.5,1,2,5,2.5,1,2,9,17,6,14,31,44.039100017307746,1.4206161295905724,15,121.11360846386408,10.5,0.09523809523809523,1271.692888870573,70.64960493725405,0.03912613122154238,9,7,4,0,2,2.25,1.75,1,0,0.5,2,9,0,0,2,7,1,4,0,2,2,0,0.5,0,2,0.5,0,1,0,0,3,0,1.5,0,3,1.5,0,2,0,0,2,1,0,2,41,10.25,0,25,1,0,1,1.4142135623730951,0.25,0,0.25,0,1,0,0,0,1,3,0,3,2,0,2,0,1,0,2,2,1,1,0,1,0,1,0,1,1,1,109.31197095345856,82.51208570936267,63.92512921254886 +greeter.py,Greeter,class,2,9,1,0.5,0,1,4,2,1,2,4,2,1,2,9,17,6,14,31,44.039100017307746,1.4206161295905724,15,121.11360846386408,10.5,0.09523809523809523,1271.692888870573,70.64960493725405,0.03912613122154238,8,7,4,0,1,2.6666666666666665,2.3333333333333335,1.3333333333333333,0,0.3333333333333333,2,8,0,0,2,7,1,4,0,1,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,3,0,1.5,0,3,1.5,0,2,0,0,2,1,0,2,41,13.666666666666666,3,25,1,0,1,1.4142135623730951,0.3333333333333333,0,0.3333333333333333,0,1,0,0,0,1,3,0,3,2,0,2,0,1,0,2,2,1,1,0,1,0,1,0,1,1,1,111.450056131092,85.49487073272815,65.17547142169123 +greeter.py,__init__,function,3,4,0,0,0,0,1,1,1,1,1,1,1,1,6,6,3,6,12,20.264662506490403,1.6887218755408668,9,38.039100017307746,6,0.16666666666666666,228.23460010384647,12.679700005769249,0.012448959890820123,2,2,1,0,0,2,2,1,0,0,2,2,0,0,2,2,1,1,0,0,1,0,1,0,1,1,1,1,0,0,2,0,2,0,2,2,2,2,0,0,0,0,0,0,13,13,13,13,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,140.6202198664881,127.27306171748977,82.2340467055486 +greeter.py,greet,function,6,9,1,1,1,1,2,2,2,2,2,2,2,2,6,9,4,7,16,23.509775004326936,1.4693609377704335,10,53.150849518197795,5.25,0.19047619047619047,279.0419599705384,15.502331109474357,0.014233938588537278,4,4,3,0,0,4,4,3,0,0,4,4,0,0,4,4,3,3,0,0,1,0,1,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,2,2,2,2,25,25,25,25,0,0,1,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,127.4217340918969,108.33349160264797,74.51563397186952 diff --git a/tests/output_formats/snapshots/output_formats__csv_test__csv_rust_counter.snap b/tests/output_formats/snapshots/output_formats__csv_test__csv_rust_counter.snap index c655dd4e..619547a2 100644 --- a/tests/output_formats/snapshots/output_formats__csv_test__csv_rust_counter.snap +++ b/tests/output_formats/snapshots/output_formats__csv_test__csv_rust_counter.snap @@ -3,6 +3,6 @@ source: tests/output_formats/csv_test.rs expression: out --- path,space_name,space_kind,start_line,end_line,cognitive.sum,cognitive.average,cognitive.min,cognitive.max,cyclomatic.sum,cyclomatic.average,cyclomatic.min,cyclomatic.max,cyclomatic.modified.sum,cyclomatic.modified.average,cyclomatic.modified.min,cyclomatic.modified.max,halstead.unique_operators,halstead.total_operators,halstead.unique_operands,halstead.total_operands,halstead.length,halstead.estimated_program_length,halstead.purity_ratio,halstead.vocabulary,halstead.volume,halstead.difficulty,halstead.level,halstead.effort,halstead.time,halstead.bugs,loc.sloc,loc.ploc,loc.lloc,loc.cloc,loc.blank,loc.sloc_average,loc.ploc_average,loc.lloc_average,loc.cloc_average,loc.blank_average,loc.sloc_min,loc.sloc_max,loc.cloc_min,loc.cloc_max,loc.ploc_min,loc.ploc_max,loc.lloc_min,loc.lloc_max,loc.blank_min,loc.blank_max,nom.functions,nom.closures,nom.functions_average,nom.closures_average,nom.total,nom.average,nom.functions_min,nom.functions_max,nom.closures_min,nom.closures_max,nargs.function_args,nargs.closure_args,nargs.function_args_average,nargs.closure_args_average,nargs.total,nargs.average,nargs.function_args_min,nargs.function_args_max,nargs.closure_args_min,nargs.closure_args_max,nexits.sum,nexits.average,nexits.min,nexits.max,tokens.sum,tokens.average,tokens.min,tokens.max,abc.assignments,abc.branches,abc.conditions,abc.magnitude,abc.assignments_average,abc.branches_average,abc.conditions_average,abc.assignments_min,abc.assignments_max,abc.branches_min,abc.branches_max,abc.conditions_min,abc.conditions_max,wmc.class_wmc_sum,wmc.interface_wmc_sum,wmc.total,npm.class_npm_sum,npm.interface_npm_sum,npm.class_methods,npm.interface_methods,npm.class_coa,npm.interface_coa,npm.total,npm.total_methods,npm.coa,npa.class_npa_sum,npa.interface_npa_sum,npa.class_attributes,npa.interface_attributes,npa.class_cda,npa.interface_cda,npa.total,npa.total_attributes,npa.cda,mi.original,mi.sei,mi.visual_studio -counter.rs,counter.rs,unit,1,13,2,2,0,2,4,4,1,2,4,4,1,2,15,23,6,15,38,74.11313393845472,1.950345629959335,21,166.9080620655929,18.75,0.05333333333333334,3129.5261637298668,173.86256465165926,0.07131777790263222,13,11,3,0,2,4.333333333333333,3.6666666666666665,1,0,0.6666666666666666,8,13,0,0,8,11,3,3,0,2,1,0,0.3333333333333333,0,1,0.3333333333333333,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,45,15,4,34,1,0,2,2.23606797749979,0.3333333333333333,0,0.6666666666666666,0,1,0,0,0,2,2,0,2,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,1,0,101.91711611092933,71.74174544054752,59.6006526964499 +counter.rs,counter.rs,unit,1,13,2,2,0,2,4,4,1,2,4,4,1,2,16,24,6,15,39,79.50977500432694,2.0387121795981265,22,173.9178331268546,20,0.05,3478.356662537092,193.24203680761622,0.07652349898080879,13,11,3,0,2,4.333333333333333,3.6666666666666665,1,0,0.6666666666666666,8,13,0,0,8,11,3,3,0,2,1,0,0.3333333333333333,0,1,0.3333333333333333,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,45,15,4,34,1,0,2,2.23606797749979,0.3333333333333333,0,0.6666666666666666,0,1,0,0,0,2,2,0,2,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,1,0,101.7031889949032,71.43311385114494,59.47554911982643 counter.rs,Counter,impl,4,13,2,2,0,2,3,3,1,2,3,3,1,2,14,20,6,13,33,68.81274391313339,2.085234664034345,20,142.62362713128297,15.166666666666666,0.06593406593406594,2163.125011491125,120.17361174950695,0.05575277534828428,10,10,3,0,0,5,5,1.5,0,0,8,10,0,0,8,10,3,3,0,0,1,0,0.5,0,1,0.5,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,38,19,4,34,1,0,2,2.23606797749979,0.5,0,1,0,1,0,0,0,2,2,0,2,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,107.21503374494837,79.28320507878054,62.69885014324466 counter.rs,bump,function,5,12,2,2,2,2,2,2,2,2,2,2,2,2,13,18,5,12,30,59.715356810271004,1.9905118936757,18,125.09775004326937,15.6,0.06410256410256411,1951.524900675002,108.41805003750011,0.05205488323759921,8,8,3,0,0,8,8,3,0,0,8,8,0,0,8,8,3,3,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,34,34,34,34,1,0,2,2.23606797749979,1,0,2,1,1,0,0,2,2,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,111.74175077815008,85.71205743468377,65.34605308663747 diff --git a/tests/parity/abc_halstead_bracket_parity.rs b/tests/parity/abc_halstead_bracket_parity.rs index 05e20477..8aa80317 100644 --- a/tests/parity/abc_halstead_bracket_parity.rs +++ b/tests/parity/abc_halstead_bracket_parity.rs @@ -45,12 +45,14 @@ use super::support::{metrics_space, ops_space}; /// /// The wildcard arm is deliberate, unlike the exhaustive `match` in /// [`super::ops_metrics_space_parity::fixture`]. Eighteen languages -/// carry the ABC bracket gate — the generic and template brackets of -/// C++, Rust, Go, Java, Groovy and the rest are the same shape — so an -/// exhaustive table would need a row for each rather than a `None` arm -/// asserting they have no such construct, which would be false. What -/// this pins is the policy, and a policy flip lands in a shared getter -/// macro or a shared ABC arm, which these seven rows already reach. +/// carry the ABC bracket gate, so an exhaustive table would need a row +/// for each rather than a `None` arm asserting they have no such +/// construct, which would be false. What this pins is the policy, and a +/// policy flip lands in a shared getter macro or a shared ABC arm, +/// which these seven rows already reach. Generic and template brackets +/// are the one family left out on purpose: Halstead bills them as a +/// single `<>` pair since #1559, which the `*_generic_angles_*` tests in +/// `src/ops.rs` pin per language. // // Two gates, one condition. The `mod` gate in `main.rs` names the seven // languages with a `Some` row and is the one that decides whether any of @@ -146,17 +148,20 @@ fn fixture(lang: LANG) -> Option<(&'static str, &'static str, &'static [&'static 22, ), // A qualified super call disambiguates its supertype with the - // same two bare tokens. `override` is in no arm of the Kotlin - // getter, which is why it is absent below. + // same two bare tokens. `override` bills as Kotlin's other + // modifiers do since #1558. // - // expected N1 = 13: the header's `class`, `:`, `()` and `{}`, - // then `fun`, the parameter `()`, the return-type `:`, the body - // `{}`, `return`, `<`, `>`, `.` and the call's `()`. + // expected N1 = 14: the header's `class`, `:`, `()` and `{}`, + // then `override`, `fun`, the parameter `()`, the return-type + // `:`, the body `{}`, `return`, `<`, `>`, `.` and the call's `()`. LANG::Kotlin => ( "class B : A() {\n override fun g(): Int { return super.g() }\n}\n", "kt", - ["()", ".", ":", "<", ">", "class", "fun", "return", "{}"].as_slice(), - 13, + [ + "()", ".", ":", "<", ">", "class", "fun", "override", "return", "{}", + ] + .as_slice(), + 14, ), // A filehandle readline. `` is deliberately not the // fixture: the grammar lexes it as one `standard_input` token, diff --git a/tests/parity/cpp_mozcpp_parity.rs b/tests/parity/cpp_mozcpp_parity.rs index 39996719..73c9f03c 100644 --- a/tests/parity/cpp_mozcpp_parity.rs +++ b/tests/parity/cpp_mozcpp_parity.rs @@ -182,13 +182,14 @@ fn cpp_and_mozcpp_agree_on_this() { ); let get = |key: &str| metric(&cpp, key); - // Operators: `{` x3, `int` x3, `;` x4, `(` x2, `return` x2, `->` x2, - // and the `*` of `S* p` -> N1 = 17. `this` must not appear here — - // classifying it as an operator (the Java / C# / Kotlin reading, - // #1380) would take this to 18 and the operand row to 9. + // Operators: `struct` (#1552), `{` x3, `int` x3, `;` x4, `(` x2, + // `return` x2, `->` x2, and the `*` of `S* p` -> N1 = 18. `this` must + // not appear here — classifying it as an operator (the Java / C# / + // Kotlin reading, #1380) would take this to 19 and the operand row + // to 9. assert_eq!( get("halstead.operators"), - 17, + 18, "`this` must not be billed as an operator: {cpp:?}" ); // Operands: `S` x2, `x` x3, `m1`, `m2`, `p` x2, `this` -> N2 = 10. diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index 0b653fcb..a737c2ff 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit 0b653fcb691c0bd2cc78296437772b044ed94719 +Subproject commit a737c2ff02d8b9884700bf5e6d024c10e14ab9d1 diff --git a/utils/check-test-lang-gates-test.py b/utils/check-test-lang-gates-test.py index 26443b92..cb8a2915 100755 --- a/utils/check-test-lang-gates-test.py +++ b/utils/check-test-lang-gates-test.py @@ -831,6 +831,404 @@ def test_three_hops_through_a_helper_that_names_nothing(self) -> None: """ self.assertEqual(_offending(source), [("shared", ["python"])]) + GUARDED_CALLS = """ +#[cfg(test)] +mod tests { + #[cfg(%s)] + fn shared() {} + + #[cfg(any(feature = "python", feature = "typescript"))] + #[test] + fn guarded_case() { + #[cfg(feature = "python")] + shared::(); + #[cfg(feature = "typescript")] + shared::(); + } + + #[cfg(feature = "rust")] + #[test] + fn rust_case() { + shared::(); + } +} +""" + + def test_a_caller_whose_every_call_is_guarded_still_feeds_the_helper( + self, + ) -> None: + """#1562, the #1556 shape: `assert_operator_rows` lacked `kotlin`. + + `guarded_case` names every parser behind an inner `#[cfg]`, so its + own needs are empty. Before #1562 that read as an always-compiled + caller, which cleared `shared`'s needs, so its gate was checked + against nothing and the first row printed OK over an `E0425` on + the `rust` leg. The second row is the guarded languages reaching + `shared` at all; the third, that the caller's own gate is not + widened by them. + """ + self.assertEqual( + _offending( + self.GUARDED_CALLS % 'any(feature = "python", feature = "typescript")' + ), + [("shared", ["rust"])], + ) + self.assertEqual( + _offending(self.GUARDED_CALLS % 'feature = "rust"'), + [("shared", ["python", "typescript"])], + ) + self.assertEqual( + _offending( + self.GUARDED_CALLS + % 'any(feature = "python", feature = "rust", feature = "typescript")' + ), + [], + ) + + def test_a_guarded_call_takes_only_its_innermost_gates_languages( + self, + ) -> None: + """The outer block names both parsers; each call is under one. + + `py_only` is correctly gated on `python`. Attributing its call to + the outer `any(…)` block as well would hand it `typescript` and + report it stranded on the `typescript` leg, where nothing calls it. + """ + source = """ +#[cfg(test)] +mod tests { + #[cfg(feature = "python")] + fn py_only() {} + + #[cfg(feature = "typescript")] + fn ts_only() {} + + #[cfg(any(feature = "python", feature = "typescript"))] + #[test] + fn nested_case() { + #[cfg(any(feature = "python", feature = "typescript"))] + { + #[cfg(feature = "python")] + py_only::(); + #[cfg(feature = "typescript")] + ts_only::(); + } + } +} +""" + items = _scan(source) + needs = gate.resolve_needs(items) + self.assertEqual(needs[_named(items, "py_only").index], {"python"}) + self.assertEqual(needs[_named(items, "ts_only").index], {"typescript"}) + self.assertEqual(gate.offenders(items, needs), []) + + GUARD_WIDER_THAN_ITS_NAMES = """ +pub struct PythonParser; +#[cfg(test)] +mod tests { + use super::*; + #[cfg(%s)] + fn shared() {} + #[test] + fn guarded_case() { + #[cfg(any(feature = "python", feature = "rust"))] + { + #[cfg(feature = "python")] + let _ = PythonParser; + shared::(); + } + } +} +""" + + def test_a_guarded_call_is_live_wherever_its_guard_holds(self) -> None: + """#1562 review: the block names only `PythonParser`; its guard admits `rust`. + + `shared::()` runs in both builds, so a `python`-only helper is + an `E0425` on the `rust` leg. Read off the parsers in the block, + the gate handed `shared` `python` alone: `--fix` wrote that gate, + exited 0, and the correct `any(python, rust)` was rejected. + """ + self.assertEqual( + _offending(self.GUARD_WIDER_THAN_ITS_NAMES % 'feature = "python"'), + [("shared", ["rust"])], + ) + self.assertEqual( + _offending( + self.GUARD_WIDER_THAN_ITS_NAMES + % 'any(feature = "python", feature = "rust")' + ), + [], + ) + + def test_a_multiline_generic_call_is_attributed_to_its_guard(self) -> None: + """#1562 review: a turbofish spread over lines is still a call. + + Matched one line at a time, `shared::<⏎ PythonParser,⏎ >()` was + in neither the guarded nor the unguarded set, so a `rust`-only + `shared` passed over an `E0425` on the `python` leg. + """ + source = """ +#[cfg(test)] +mod tests { + #[cfg(%s)] + fn shared() {} + #[test] + fn guarded_case() { + #[cfg(feature = "python")] + { + shared::< + PythonParser, + >(); + } + } + #[cfg(feature = "rust")] + #[test] + fn rust_case() { shared::(); } +} +""" + self.assertEqual( + _offending(source % 'feature = "rust"'), [("shared", ["python"])] + ) + self.assertEqual( + _offending(source % 'any(feature = "python", feature = "rust")'), [] + ) + + def test_an_all_guard_hands_over_the_parsers_its_block_names(self) -> None: + """No single-language build admits `all(python, rust)`. + + The call is made only where both are on, so the predicate names + no build to hand over; the parser in the block still says what + the helper needs. Read as unconditional instead, `shared` lost + its needs and the ungated helper passed. + """ + source = """ +#[cfg(test)] +mod tests { + fn shared() {} + #[test] + fn guarded_case() { + #[cfg(all(feature = "python", feature = "rust"))] + { + shared::(); + } + } +} +""" + self.assertEqual(_offending(source), [("shared", ["python"])]) + + def test_a_nested_items_own_gate_is_not_a_guard_on_its_container( + self, + ) -> None: + """A `rust`-gated test inside `mod tests` hands `rust` to nothing. + + The test names no parser, so it needs nothing of `fixtures`; it + is the test's own `#[cfg]`, not a guard on a call the `mod` + makes. Read as a guard, the `mod` handed `fixtures` `rust`, + `fixtures` passed it on to `PYTHON`, and a correctly `python`-gated + table read as stranded on the `rust` leg -- the shape of + `src/spaces_tests.rs`. + """ + source = """ +#[cfg(test)] +mod tests { + #[cfg(feature = "python")] + const PYTHON: &str = ""; + + fn fixtures() -> Vec<(LANG, &'static str)> { + vec![ + #[cfg(feature = "python")] + (LANG::Python, PYTHON), + ] + } + + #[cfg(feature = "rust")] + #[test] + fn rust_case() { + let _ = fixtures(); + } +} +""" + self.assertEqual(_offending(source), []) + + NESTED_ARM = """ +#[cfg(test)] +mod tests { + #[cfg(feature = "python")] + fn shared() {} + + #[cfg(any(feature = "python", feature = "typescript"))] + #[test] + fn nested_arm_case() { + #[cfg(%s)] + { + shared(); + %s + check::(); + #[cfg(feature = "typescript")] + check::(); + } + } +} +""" + + def test_a_call_beside_a_nested_arm_does_not_take_the_arms_languages( + self, + ) -> None: + """The arm sits inside the call's innermost extent, not around it. + + `shared()` is made only where `python` is, so a `python`-gated + helper is right. Unioning the extent's lines handed it the nested + arm's `typescript` too, and reported it stranded on that leg. The + second row is the call made directly in an `any(...)` block, live + on both legs: a `python`-only helper there is a real `E0425`. + """ + self.assertEqual( + _offending(self.NESTED_ARM % ('feature = "python"', "")), + [], + ) + self.assertEqual( + _offending( + self.NESTED_ARM + % ( + 'any(feature = "python", feature = "typescript")', + '#[cfg(feature = "python")]', + ) + ), + [("shared", ["typescript"])], + ) + + GUARD_NAMING_NO_PARSER = """ +#[cfg(test)] +mod tests { + %s + fn shared() {} + + #[test] + fn plain() { + %s + shared(); + } + + #[cfg(feature = "rust")] + #[test] + fn rust_case() { + check_metrics::("fn f() {}", "f.rs", |m| {}); + shared(); + } +} +""" + + def test_a_guard_holding_with_no_language_keeps_the_call( + self, + ) -> None: + """A guard true in the no-language build admits the call everywhere. + + `plain` is in every build and calls `shared` wherever its guard + holds, and these hold with every grammar off, so the call stays + unconditional: a `rust`-only `shared` is gated out from under it + (`E0425`) -- the gate must refuse that and accept the ungated + helper, as it did before #1562. + """ + for guard in ( + '#[cfg(feature = "vcs-git")]', + '#[cfg(not(feature = "python"))]', + ): + items = _scan(self.GUARD_NAMING_NO_PARSER % ("", guard)) + needs = gate.resolve_needs(items) + self.assertEqual(needs[_named(items, "shared").index], set(), guard) + self.assertEqual(gate.offenders(items, needs), [], guard) + narrow = _scan( + self.GUARD_NAMING_NO_PARSER % ('#[cfg(feature = "rust")]', guard) + ) + self.assertEqual( + [ + (i.name, sorted(w)) + for i, w in gate.over_gated(narrow, gate.resolve_needs(narrow)) + ], + [("shared", ["rust"])], + guard, + ) + + def test_a_language_guard_hands_its_builds_without_naming_a_parser( + self, + ) -> None: + """#1562 review: the guard says where the call is made, not the names. + + `plain` names no parser, but its call runs only in the builds its + guard admits, so `shared` needs those beside `rust_case`'s `rust`. + Read off the names, an empty set left the call unconditional and + the helper ungated, dead code on every leg nothing calls it from. + The `any(…)` of exactly those builds is the gate that compiles. + """ + for guard, builds in ( + ('#[cfg(feature = "python")]', ["python"]), + ( + '#[cfg(any(feature = "python", feature = "typescript"))]', + ["python", "typescript"], + ), + ('#[cfg(all(feature = "python", feature = "vcs-git"))]', ["python"]), + ): + items = _scan(self.GUARD_NAMING_NO_PARSER % ("", guard)) + needs = gate.resolve_needs(items) + self.assertEqual( + sorted(needs[_named(items, "shared").index]), sorted(builds + ["rust"]) + ) + narrow = '#[cfg(feature = "rust")]' + self.assertEqual( + _offending(self.GUARD_NAMING_NO_PARSER % (narrow, guard)), + [("shared", builds)], + guard, + ) + exact = "#[cfg(any(%s))]" % ", ".join( + f'feature = "{b}"' for b in sorted(builds + ["rust"]) + ) + self.assertEqual( + _offending(self.GUARD_NAMING_NO_PARSER % (exact, guard)), [], guard + ) + + GUARDED_REFERENCES = """ +#[cfg(test)] +mod tests { + #[cfg(any(feature = "python", feature = "typescript"))] + fn shared(_: &[u8]) {} + + #[cfg(%s)] + const ROWS: &[u8] = &[]; + + #[cfg(any(feature = "python", feature = "typescript"))] + #[test] + fn guarded_case() { + #[cfg(feature = "python")] + shared::(ROWS); + #[cfg(feature = "typescript")] + shared::(ROWS); + } +} +""" + + def test_a_table_used_only_under_guarded_calls_takes_their_languages( + self, + ) -> None: + """#1562's `const` half: `C_TYPEDEF_ROWS` beside `assert_operator_rows`. + + Every use of `ROWS` sits under an inner `#[cfg]`. Read as a use by + an always-compiled caller, that cleared its needs, so a gate + missing `typescript` printed OK over an `E0425` on that leg, and + the over-gate check misreported `python` as never used. + """ + source = self.GUARDED_REFERENCES % 'feature = "python"' + self.assertEqual(_offending(source), [("ROWS", ["typescript"])]) + items = _scan(source) + self.assertEqual(gate.over_gated(items, gate.resolve_needs(items)), []) + self.assertEqual( + _offending( + self.GUARDED_REFERENCES + % 'any(feature = "python", feature = "typescript")' + ), + [], + ) + def test_a_cycle_terminates_and_reports_each_narrow_link(self) -> None: """Each half of the cycle is reached from the other's test. diff --git a/utils/check-test-lang-gates.py b/utils/check-test-lang-gates.py index 3f4d9006..69ef86e2 100755 --- a/utils/check-test-lang-gates.py +++ b/utils/check-test-lang-gates.py @@ -31,7 +31,9 @@ concrete ``*Parser`` aliases and ``LANG::`` literals in its body, ignoring anything already behind a nested ``#[cfg(feature = …)]``, plus transitively the needed set of - same-module helper ``fn``s it calls. + same-module helper ``fn``s it calls. A call *inside* such a nested + gate hands the languages named under it to the callee instead, so a + test whose every parser is guarded still gates its helpers (#1562). 2. **Declared gate** -- the item's own ``cfg`` conjoined with every enclosing ``mod``'s. 3. **Verdict** -- for each needed feature, evaluate the declared gate @@ -66,6 +68,7 @@ from __future__ import annotations import argparse +import bisect import os import shutil import pathlib @@ -815,8 +818,21 @@ class Item: hand_written: frozenset[str] #: Call-shaped references in the body, for resolving `fn`s. calls: frozenset[str] + #: The subset of `calls` made outside every inner `#[cfg(feature = …)]`, + #: so in each build the item itself is compiled into. + unguarded_calls: frozenset[str] + #: `(callee, languages)` for each call made inside an inner + #: `#[cfg(feature = …)]`, with the languages named under that same + #: gate. The callee is live wherever the guarded call is (#1562). + guarded_calls: tuple[tuple[str, frozenset[str]], ...] #: Every identifier in the body, for resolving `const` / `type`. references: frozenset[str] + #: `references` and `guarded_calls` again, for a referent with no + #: call shape (a `const` fixture table): what sits outside every + #: inner `#[cfg(feature = …)]`, and `(name, languages)` per + #: identifier inside one. + unguarded_references: frozenset[str] + guarded_references: tuple[tuple[str, frozenset[str]], ...] parent_index: int | None #: Index into the file's item list; the parent link is by index so #: the dataclass stays comparable and printable. @@ -1061,6 +1077,12 @@ def scan_source( macro_end ].count("}") macro_body = "\n".join(lines[index : macro_end + 1]) + macro_calls = frozenset( + m.group(1) for m in CALL_RE.finditer(macro_body) + ) + macro_references = frozenset( + m.group(1) for m in REFERENCE_RE.finditer(macro_body) + ) items.append( Item( path=path, @@ -1090,12 +1112,12 @@ def scan_source( direct=frozenset(), hardcoded=frozenset(), hand_written=_hand_written_features(raw, attr_line), - calls=frozenset( - m.group(1) for m in CALL_RE.finditer(macro_body) - ), - references=frozenset( - m.group(1) for m in REFERENCE_RE.finditer(macro_body) - ), + calls=macro_calls, + unguarded_calls=macro_calls, + guarded_calls=(), + references=macro_references, + unguarded_references=macro_references, + guarded_references=(), parent_index=stack[-1][0] if stack else None, index=len(items), in_test_scope=( @@ -1195,9 +1217,12 @@ def scan_source( # the test got no gate and panicked without the grammar — the # unsafe direction. guarded: dict[int, list[str]] = {} - for first, last, predicate in _inner_cfg_extents(lines, raw, index, body_end): + extents = _inner_cfg_extents(lines, raw, index, body_end) + for first, last, predicate in extents: for guarded_line in range(first, last + 1): guarded.setdefault(guarded_line, []).append(predicate) + # The languages each guarded line names, for `guarded_calls`. + skipped: dict[int, set[str]] = {} # A helper that maps one of its arguments onto a `LANG` is a # dispatcher, and its arms are options rather than requirements. # `tests/api/suppression_test.rs`'s `analyze_lang` picks the @@ -1247,11 +1272,100 @@ def scan_source( continue feature = table[hit.group(0)] if any(not gate_admits(g, feature) for g in inner_gates): + skipped.setdefault(line_number, set()).add(feature) continue direct.add(feature) if not hit.group(0).startswith("LANG::"): hardcoded.add(feature) body_offset += len(lines[line_number]) + 1 + # A call under `#[cfg(feature = "javascript")]` is made only in a + # build with that grammar, so it is not an unconditional use of + # its callee. It is still a use: before #1562 the languages it + # named were dropped as "already conditional" and nothing handed + # them to the callee, so a test whose every parser sat behind an + # inner gate read as naming nothing, was taken for always + # compiled, and cleared the needs of the helper it called. That + # helper then had no derived gate at all, and one missing a + # caller's language passed while failing to compile (E0425). + # + # What a guarded call hands its callee is the set of builds its + # guards admit, read off the predicates rather than off the + # parsers named near it. Reading the names made a call in an + # `any(python, rust)` block that names only `PythonParser` hand + # over `python` alone, so `--fix` gated the helper out of the + # `rust` build the call is also made in, and printed OK over the + # `E0425` it caused (#1562 review). The guards are every extent + # around the line, so `a` in `#[cfg(any(js, ts))] { #[cfg(js)] + # a(); }` takes `js` alone, not the outer block's `ts` as well. + # + # A guard that holds in the build with no language at all + # (`vcs-git`, `not(python)`) admits the call wherever the caller + # is, so the line stays an unconditional use: dropping it from + # both sets left the callee gated on its other callers alone, and + # `E0425` on that leg. An `all(…)` of languages admits no + # single-language build, so it hands over what its block names, + # as before; with nothing named it too stays unconditional. + # + # A `const` table passed to a guarded call is live exactly where + # that call is, so the line's references are attributed the same + # way as its calls. + # + # Every line of one innermost extent is under the same guards, so + # the answer is computed once per extent; per line, rescanning + # every extent made the gate quadratic in a test module's gated + # items. + # Smallest first, so `setdefault` keeps each line's innermost + # extent; a stable sort settles a tie as `min` would, by order. + innermost: dict[int, tuple[int, int]] = {} + for first, last, _ in sorted(extents, key=lambda extent: extent[1] - extent[0]): + for row in range(first, last + 1): + innermost.setdefault(row, (first, last)) + attribution: dict[tuple[int, int], frozenset[str] | None] = {} + for n in guarded: + span = innermost[n] + if span in attribution: + continue + + def admits(disabled: frozenset[str], gates: list[str] = guarded[n]) -> bool: + return all(evaluate_predicate(g, disabled) for g in gates) + + first, last = span + # A nested item's own `#[cfg]` is its gate, not a guard on a + # call this item makes: the nested item derives its own + # needs from its own calls. Read as a guard, a `rust`-gated + # test that names no parser handed `rust` through its `mod` + # to every helper it calls. + if ITEM_RE.match(lines[first]) or admits(KNOWN_FEATURES): + attribution[span] = None + continue + live = frozenset( + f for f in KNOWN_FEATURES if admits(KNOWN_FEATURES - enabled_closure(f)) + ) + named = frozenset().union( + *(skipped.get(k, ()) for k in range(first, last + 1)) + ) + attribution[span] = live or named or None + # Calls are matched over the whole body and placed by where they + # start: matched per line, a call spread over several lines + # (`shared::<\n PythonParser,\n>()`) was in neither set. + line_starts = [0] + for n in range(index, body_end): + line_starts.append(line_starts[-1] + len(lines[n]) + 1) + unguarded_calls: set[str] = set() + unguarded_references: set[str] = set() + guarded_calls: list[tuple[str, frozenset[str]]] = [] + guarded_references: list[tuple[str, frozenset[str]]] = [] + for pattern, unguarded_set, guarded_list in ( + (CALL_RE, unguarded_calls, guarded_calls), + (REFERENCE_RE, unguarded_references, guarded_references), + ): + for hit in pattern.finditer(body): + n = index + bisect.bisect_right(line_starts, hit.start()) - 1 + languages = attribution[innermost[n]] if n in guarded else None + if languages is None: + unguarded_set.add(hit.group(1)) + else: + guarded_list.append((hit.group(1), languages)) item = Item( path=path, @@ -1276,7 +1390,11 @@ def scan_source( hardcoded=frozenset(hardcoded), hand_written=_hand_written_features(raw, attr_line), calls=frozenset(m.group(1) for m in CALL_RE.finditer(body)), + unguarded_calls=frozenset(unguarded_calls), + guarded_calls=tuple(guarded_calls), references=frozenset(m.group(1) for m in REFERENCE_RE.finditer(body)), + unguarded_references=frozenset(unguarded_references), + guarded_references=tuple(guarded_references), parent_index=parent_index, index=len(items), in_test_scope=in_test_scope, @@ -1584,6 +1702,27 @@ def reachable(item: Item, generic_too: bool) -> set[str]: if not gate_excludes(predicate, feature) } + # A call behind an inner `#[cfg]` hands its callee the languages named + # under that gate, and nothing to the caller's own `all(...)`: the + # caller is compiled without them, the call is not (#1562). Seeded + # here so the fixpoint below carries it on to the callee's callees. + for item in items: + predicate = effective_predicate(items, item) + for name, languages in item.guarded_calls: + callee_index = _resolve_call(items, scopes, item, name) + if callee_index is None or items[callee_index].is_test_fn: + continue + needs[callee_index] |= { + f for f in languages if not gate_excludes(predicate, f) + } + # The same for a referent with no call shape: a `const` table + # passed to a guarded call is live exactly where that call is. + for name, languages in item.guarded_references: + index = _resolve_call(items, scopes, item, name) + if index is None or items[index].kind == "fn" or items[index].is_test_fn: + continue + needs[index] |= {f for f in languages if not gate_excludes(predicate, f)} + # A referent with even one unconditional caller must stay # unconditional itself, whatever its other callers need. Without # this the reverse propagation below gates a helper on the union of @@ -1642,6 +1781,15 @@ def reachable(item: Item, generic_too: bool) -> set[str]: continue if not _uses(item, items[index], strict=True): continue + # A call behind an inner `#[cfg]` is made only where that + # gate holds, so it makes nothing always-compiled (#1562). + unguarded = ( + item.unguarded_calls + if items[index].kind == "fn" + else item.unguarded_references + ) + if name not in unguarded: + continue unconditional.add(index) needs[index] = set() changed = True