From 4b8f24c8e4d203481decb7c936e4e2998eed05ad Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 09:34:08 -0700 Subject: [PATCH 01/48] refactor(abc): share the chain-operand and argument walks #1526 gave the boolean-slot rule one home, but every language still carried its own `_count_chain_operands` reading `left`/`right` and its own negated-argument loop, so a change to the operand rule had to be made in about a dozen places (lesson 59). Add `count_field_operands` (Rule 9 over a `left`/`right` chain) and `count_each_operand` (every operand child of a list) to abc.rs, each taking the language's slot scorer. The nine field-reading copies and the argument/return-list loops now call them; per-language rationale moved to the call sites. Each caller keeps its own parent gate, so the change is behaviour-neutral: no snapshot moved. Refs #1537 --- src/metrics/abc.rs | 28 +++++++++++++++++++- src/metrics/abc/cpp.rs | 16 +++++------- src/metrics/abc/csharp.rs | 27 ++++++------------- src/metrics/abc/elixir.rs | 22 ++++------------ src/metrics/abc/groovy.rs | 16 ++++-------- src/metrics/abc/irules.rs | 8 +++--- src/metrics/abc/java.rs | 29 ++++++--------------- src/metrics/abc/js_family.rs | 50 ++++++++++++++++-------------------- src/metrics/abc/kotlin.rs | 20 ++++----------- src/metrics/abc/lua.rs | 42 +++++++++++------------------- src/metrics/abc/perl.rs | 12 +++------ src/metrics/abc/php.rs | 22 ++++++---------- src/metrics/abc/python.rs | 20 ++++----------- src/metrics/abc/ruby.rs | 27 ++++++------------- src/metrics/abc/rust.rs | 24 +++++------------ src/metrics/abc/tcl.rs | 8 +++--- 16 files changed, 140 insertions(+), 231 deletions(-) diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index de0c449d..9654df42 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -527,6 +527,32 @@ pub(super) fn count_negated_operand<'a>( } } +// A language's scorer for one operand: its `count_boolean_slot` or +// `count_negated_operand` wrapper. +pub(super) type OperandScorer = fn(&Node, &mut f64); + +// Fitzpatrick Rule 9 for a chain whose grammar names its two operands +// `left` and `right`: each is scored by `score`. Read by field rather +// than as every operand child, so a comment beside the operator — a +// named child too — does not pay. The caller owns the gate deciding +// that `chain` is a chain at all. +pub(super) fn count_field_operands(chain: &Node, score: OperandScorer, conditions: &mut f64) { + for field in ["left", "right"] { + if let Some(operand) = chain.child_by_field_name(field) { + score(&operand, conditions); + } + } +} + +// Each operand child of `list` — an argument list, a `return` value +// list, a chain whose grammar names no operand field — scored by +// `score`. `is_operand` keeps punctuation and comments out. +pub(super) fn count_each_operand(list: &Node, score: OperandScorer, conditions: &mut f64) { + for operand in list.children().filter(is_operand) { + score(&operand, conditions); + } +} + // Default no-op `Abc` impls. Audited in #188; the matrix below // records the rationale for every entry so the no-op default is a // deliberate choice, not scaffolding leftover. @@ -7294,7 +7320,7 @@ function f(int $a, int $b): int { // The wrapper peel has two callers, and #1459's fixture members all // reach it through the *slot* (`kotlin_count_condition`). This is the - // other one: the `&&` / `||` walker (`kotlin_count_chain_operands`), + // other one: the `&&` / `||` walker (`count_field_operands`), // which hands each operand to the same slot. The two wrappers #1459 // taught it therefore have a // second, structurally independent path into the count, and a diff --git a/src/metrics/abc/cpp.rs b/src/metrics/abc/cpp.rs index 56ff03e8..e531e266 100644 --- a/src/metrics/abc/cpp.rs +++ b/src/metrics/abc/cpp.rs @@ -9,7 +9,10 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, wrapped_operand}; +use super::{ + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, is_operand, wrapped_operand, +}; use crate::metrics::cyclomatic::cpp_operator_is_applied; use crate::*; @@ -142,9 +145,7 @@ pub(super) fn cpp_count_return(statement: &Node, conditions: &mut f64) { // Each operand of a call's `argument_list` or of an Objective-C message // is a negated operand (`f(!a, !b)`). pub(super) fn cpp_count_arguments(list: &Node, conditions: &mut f64) { - for operand in list.children().filter(is_operand) { - cpp_count_negated(&operand, conditions); - } + count_each_operand(list, cpp_count_negated, conditions); } // Fitzpatrick Rule 9 (C++ in Figure 3, #403): each operand of an `&&` / @@ -167,12 +168,7 @@ pub(super) fn cpp_count_arguments(list: &Node, conditions: &mut f64) { // arms do, scores nothing. pub(super) fn cpp_count_chain_operands(chain: &Node, conditions: &mut f64) { match chain.kind() { - "binary_expression" => { - let operands = ["left", "right"].map(|field| chain.child_by_field_name(field)); - for operand in operands.iter().flatten() { - cpp_count_condition(operand, conditions); - } - } + "binary_expression" => count_field_operands(chain, cpp_count_condition, conditions), "fold_expression" => { let mut written = 0; for operand in chain.children().filter(is_operand) { diff --git a/src/metrics/abc/csharp.rs b/src/metrics/abc/csharp.rs index bde8c98f..66b1b7b9 100644 --- a/src/metrics/abc/csharp.rs +++ b/src/metrics/abc/csharp.rs @@ -10,8 +10,8 @@ )] use super::{ - Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, last_operand, peel, - wrapped_operand, + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, last_operand, peel, wrapped_operand, }; use crate::macros::{csharp_paren_expr_kinds, csharp_prefix_unary_expr_kinds}; use crate::*; @@ -166,22 +166,7 @@ fn csharp_count_negated_slot(slot: Option, conditions: &mut f64) { // Each argument of a call is a negated operand. fn csharp_count_arguments(arguments: &Node, conditions: &mut f64) { - for argument in arguments.children().filter(is_operand) { - csharp_count_negated(&argument, conditions); - } -} - -// Fitzpatrick Rule 9: each operand of an `&&` / `||` chain is a boolean -// slot. `a && b || c` is a left-nested chain of `binary_expression`s, so -// an operand that is itself a chain is paid by its own operator's visit. -// Read by field: a comment beside the operator is a named child too, and -// must not pay. -fn csharp_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - csharp_count_condition(&operand, conditions); - } - } + count_each_operand(arguments, csharp_count_negated, conditions); } // ABC token-level helpers for C#. Mirror of Java's helper layout with @@ -632,12 +617,16 @@ fn csharp_walk_for_conditions<'a>( use Csharp::*; let conds = &mut stats.conditions; match node.kind_id().into() { + // Fitzpatrick Rule 9: each operand of an `&&` / `||` chain is a + // boolean slot. `a && b || c` is a left-nested chain of + // `binary_expression`s, so an operand that is itself a chain is + // paid by its own operator's visit. AMPAMP | PIPEPIPE => { if let Some(chain) = ancestors .parent(node) .filter(|p| matches!(p.kind_id().into(), BinaryExpression | BinaryExpression2)) { - csharp_count_chain_operands(&chain, conds); + count_field_operands(&chain, csharp_count_condition, conds); } } // `compute` returns as soon as `csharp_count_token_branch` fires, diff --git a/src/metrics/abc/elixir.rs b/src/metrics/abc/elixir.rs index 98db89f2..fe3dc89c 100644 --- a/src/metrics/abc/elixir.rs +++ b/src/metrics/abc/elixir.rs @@ -9,7 +9,7 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, last_operand, wrapped_operand}; +use super::{Abc, Stats, count_boolean_slot, count_field_operands, last_operand, wrapped_operand}; use crate::lang_helpers::elixir::elixir_call_keyword; use crate::*; @@ -191,20 +191,6 @@ fn elixir_has_else(call: &Node, code: &[u8]) -> bool { }) } -// Each operand of an `&&` / `||` / `and` / `or` chain is a boolean slot -// (Fitzpatrick Rule 9). tree-sitter-elixir parses `a && b || c` as a -// left-nested chain of `binary_operator`s, so an operand that is itself -// a chain is paid by its own operator's visit. Read by field: a comment -// beside the operator is a named child of the `binary_operator` too, -// and must not pay. -fn elixir_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - elixir_count_condition(&operand, conditions); - } - } -} - // What an Elixir `Call` contributes. The classification is by keyword // text rather than by kind, so it is a paragraph of policy rather than a // dispatch arm — the same split `java_count_token_branch` and @@ -460,13 +446,15 @@ impl Abc for ElixirCode { // The short-circuit operators are not counted directly // (cross-language policy, #395). Under error recovery the // token's parent can be an `ERROR` node, which has no - // operands to score. + // operands to score. `a && b || c` is a left-nested chain of + // `binary_operator`s, so an operand that is itself a chain is + // paid by its own operator's visit. E::AMPAMP | E::PIPEPIPE | E::And | E::Or => { if let Some(chain) = ancestors .parent(node) .filter(|parent| parent.kind() == BINARY_OPERATOR) { - elixir_count_chain_operands(&chain, &mut stats.conditions); + count_field_operands(&chain, elixir_count_condition, &mut stats.conditions); } } E::StabClause if npa::elixir_clause_is_decision(node, code, ancestors) => { diff --git a/src/metrics/abc/groovy.rs b/src/metrics/abc/groovy.rs index c67e1170..9335365a 100644 --- a/src/metrics/abc/groovy.rs +++ b/src/metrics/abc/groovy.rs @@ -10,7 +10,8 @@ )] use super::{ - Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, peel, wrapped_operand, + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, peel, wrapped_operand, }; use crate::*; @@ -366,23 +367,16 @@ fn groovy_walk_for_conditions<'a>( use Groovy::*; let conds = &mut stats.conditions; match node.kind_id().into() { - // Each operand of an `&&` / `||` chain is a boolean slot, read by - // field so a comment beside the operator does not pay. + // Each operand of an `&&` / `||` chain is a boolean slot. AMPAMP | PIPEPIPE => { if let Some(chain) = ancestors .parent(node) .filter(|p| p.kind_id() == BinaryExpression) { - for field in ["left", "right"] { - groovy_count_slot(chain.child_by_field_name(field), conds); - } - } - } - ArgumentList => { - for argument in node.children().filter(is_operand) { - groovy_count_negated(&argument, conds); + count_field_operands(&chain, groovy_count_condition, conds); } } + ArgumentList => count_each_operand(node, groovy_count_negated, conds), VariableDeclarator => groovy_count_negated_slot(node.child_by_field_name("value"), conds), AssignmentExpression => { groovy_count_negated_slot(node.child_by_field_name("right"), conds); diff --git a/src/metrics/abc/irules.rs b/src/metrics/abc/irules.rs index 163b77e1..58b8d6e8 100644 --- a/src/metrics/abc/irules.rs +++ b/src/metrics/abc/irules.rs @@ -9,7 +9,9 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, wrapped_operand}; +use super::{ + Abc, Stats, count_boolean_slot, count_each_operand, count_negated_operand, wrapped_operand, +}; use crate::*; /// The three operand slots of a `ternary_expr`, located relative to the @@ -244,8 +246,6 @@ fn irules_count_condition(condition: &Node, conditions: &mut f64) { // operand field, and inlines the parens of `($a)`. fn irules_count_chain_operands(chain: &Node, conditions: &mut f64) { if chain.kind_id() == Irules::BinopExpr { - for operand in chain.children().filter(is_operand) { - irules_count_condition(&operand, conditions); - } + count_each_operand(chain, irules_count_condition, conditions); } } diff --git a/src/metrics/abc/java.rs b/src/metrics/abc/java.rs index b01444c1..c154116c 100644 --- a/src/metrics/abc/java.rs +++ b/src/metrics/abc/java.rs @@ -10,7 +10,8 @@ )] use super::{ - Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, peel, wrapped_operand, + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, peel, wrapped_operand, }; use crate::*; @@ -96,19 +97,6 @@ fn java_count_negated(operand: &Node, conditions: &mut f64) { ); } -// Fitzpatrick Rule 9 (#403): each operand of an `&&` / `||` chain is a -// boolean slot. `a && b || c` is a left-nested chain of -// `binary_expression`s, so an operand that is itself a chain is paid by -// its own operator's visit. Read by field: a comment beside the operator -// is a named child too, and must not pay. -fn java_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - java_count_condition(&operand, conditions); - } - } -} - fn java_count_slot(slot: Option, conditions: &mut f64) { if let Some(condition) = slot { java_count_condition(&condition, conditions); @@ -311,21 +299,20 @@ fn java_walk_for_conditions<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>, s use Java::*; let conds = &mut stats.conditions; match node.kind_id().into() { - // Each operand of an `&&` / `||` chain is a boolean slot. + // Fitzpatrick Rule 9 (#403): each operand of an `&&` / `||` chain + // is a boolean slot. `a && b || c` is a left-nested chain of + // `binary_expression`s, so an operand that is itself a chain is + // paid by its own operator's visit. AMPAMP | PIPEPIPE => { if let Some(chain) = ancestors .parent(node) .filter(|p| p.kind_id() == BinaryExpression) { - java_count_chain_operands(&chain, conds); + count_field_operands(&chain, java_count_condition, conds); } } // Negated operands among method arguments. - ArgumentList => { - for argument in node.children().filter(is_operand) { - java_count_negated(&argument, conds); - } - } + ArgumentList => count_each_operand(node, java_count_negated, conds), // `if (cond)`, `while (cond)`, `do … while (cond);`, by grammar // field: a fixed index lands on a comment before the slot // (`if /*c*/ (b)`) and scores the condition zero (#1455). diff --git a/src/metrics/abc/js_family.rs b/src/metrics/abc/js_family.rs index dea1f561..ec0aba7b 100644 --- a/src/metrics/abc/js_family.rs +++ b/src/metrics/abc/js_family.rs @@ -10,8 +10,8 @@ )] use super::{ - Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, last_operand, - wrapped_operand, + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, last_operand, wrapped_operand, }; use crate::*; @@ -104,19 +104,6 @@ fn js_family_count_negated(operand: &Node, conditions: &mut f64) { ); } -// Fitzpatrick Rule 9 (#403): each operand of an `&&` / `||` chain is a -// boolean slot. `a && b || c` is a left-nested chain of -// `binary_expression`s, so an operand that is itself a chain is paid by -// its own operator's visit. Read by field: a comment beside the operator -// is a named child too, and must not pay. -fn js_family_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - js_family_count_condition(&operand, conditions); - } - } -} - // Phase-2B (issues #403 / #1102): a ternary's condition is a boolean // slot, and each branch operand a negated operand, exactly as // `java_walk_ternary` counts them. Without this the JS family scored @@ -160,13 +147,6 @@ fn js_family_walk_for(node: &Node, conditions: &mut f64) { } } -// Each argument of a call is a negated operand (`f(!a, !b)`). -fn js_family_count_arguments(arguments: &Node, conditions: &mut f64) { - for argument in arguments.children().filter(is_operand) { - js_family_count_negated(&argument, conditions); - } -} - // Generates the per-language predicate deciding whether an `=` token // initialises a `const` binding, whose initializer is part of the // declaration and therefore not an ABC assignment (Fitzpatrick). @@ -367,10 +347,17 @@ macro_rules! ts_abc_compute { stats.conditions += 1.; } // Fitzpatrick Rule 9: each operand of a `&&` / `||` - // chain is one condition (issue #403). + // chain is one condition (issue #403). `a && b || c` is a + // left-nested chain of `binary_expression`s, so an operand + // that is itself a chain is paid by its own operator's + // visit. AMPAMP | PIPEPIPE => { if let Some(chain) = ancestors.parent(node) { - js_family_count_chain_operands(&chain, &mut stats.conditions); + count_field_operands( + &chain, + js_family_count_condition, + &mut stats.conditions, + ); } } // Phase-2B (issue #403): condition slots. JS / TS @@ -395,7 +382,7 @@ macro_rules! ts_abc_compute { } // Method-argument walker for `f(!a, !b)`. Arguments => { - js_family_count_arguments(node, &mut stats.conditions); + count_each_operand(node, js_family_count_negated, &mut stats.conditions); } // `a ? !b : !c` — the ternary's own `?` token is // already counted by the condition arm above; this @@ -488,10 +475,17 @@ macro_rules! js_abc_compute { stats.conditions += 1.; } // Fitzpatrick Rule 9: each operand of a `&&` / `||` - // chain is one condition (issue #403). + // chain is one condition (issue #403). `a && b || c` is a + // left-nested chain of `binary_expression`s, so an operand + // that is itself a chain is paid by its own operator's + // visit. AMPAMP | PIPEPIPE => { if let Some(chain) = ancestors.parent(node) { - js_family_count_chain_operands(&chain, &mut stats.conditions); + count_field_operands( + &chain, + js_family_count_condition, + &mut stats.conditions, + ); } } // Phase-2B (issue #403): condition slots. Same shape @@ -508,7 +502,7 @@ macro_rules! js_abc_compute { } } Arguments => { - js_family_count_arguments(node, &mut stats.conditions); + count_each_operand(node, js_family_count_negated, &mut stats.conditions); } // `a ? !b : !c` — the ternary's own `?` token is // already counted by the condition arm above; this diff --git a/src/metrics/abc/kotlin.rs b/src/metrics/abc/kotlin.rs index b69b159e..04154e4f 100644 --- a/src/metrics/abc/kotlin.rs +++ b/src/metrics/abc/kotlin.rs @@ -9,7 +9,7 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, wrapped_operand}; +use super::{Abc, Stats, count_boolean_slot, count_field_operands, wrapped_operand}; use crate::*; // Fitzpatrick's ABC rules adapted for Kotlin syntax. Kotlin shares the @@ -175,18 +175,6 @@ fn kotlin_count_condition(condition: &Node, conditions: &mut f64) { ); } -// Fitzpatrick Rule 9 (#557): each operand of an `&&` / `||` chain is a -// boolean slot. `a && b || c` is a left-nested chain of -// `binary_expression`s, so an operand that is itself a chain is paid by -// its own operator's visit. Read by field, so a comment beside the -// operator does not pay. -fn kotlin_count_chain_operands(chain: &Node, conditions: &mut f64) { - let operands = ["left", "right"].map(|field| chain.child_by_field_name(field)); - for operand in operands.iter().flatten() { - kotlin_count_condition(operand, conditions); - } -} - // Returns true when the `when_expression` enclosing `entry` carries a // subject — `when (x) { … }` rather than `when { … }`. The distinction // decides how the entry's condition is scored (#1421): a subject-ful @@ -484,10 +472,12 @@ impl Abc for KotlinCode { // `&&` / `||` chain is one condition (issue #557). The short- // circuit operators are not counted directly (cross-language // policy, #395); the walker fires off the operator token and - // inspects the parent `binary_expression`. + // inspects the parent `binary_expression`. `a && b || c` is a + // left-nested chain, so an operand that is itself a chain is + // paid by its own operator's visit. AMPAMP | PIPEPIPE => { if let Some(parent) = ancestors.parent(node) { - kotlin_count_chain_operands(&parent, &mut stats.conditions); + count_field_operands(&parent, kotlin_count_condition, &mut stats.conditions); } } _ => {} diff --git a/src/metrics/abc/lua.rs b/src/metrics/abc/lua.rs index 179c96a5..1324a6f4 100644 --- a/src/metrics/abc/lua.rs +++ b/src/metrics/abc/lua.rs @@ -9,7 +9,10 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, wrapped_operand}; +use super::{ + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, wrapped_operand, +}; use crate::*; // Fitzpatrick's ABC rules adapted for Lua. @@ -103,27 +106,6 @@ fn lua_count_slot(slot: Option, conditions: &mut f64) { } } -// Each operand of an `and` / `or` chain is a boolean slot (Fitzpatrick -// Rule 9, #403). `a and b or c` is a left-nested chain of -// `binary_expression`s, so an operand that is itself a chain is paid by -// its own operator's visit. Read by field: a comment beside the operator -// is a named child too, and must not pay. -fn lua_count_chain_operands(chain: &Node, conditions: &mut f64) { - if chain.kind_id() == Lua::BinaryExpression { - for field in ["left", "right"] { - lua_count_slot(chain.child_by_field_name(field), conditions); - } - } -} - -// Each operand of an `expression_list` or `arguments` list is a -// negated operand. -fn lua_count_negated_operands(list: &Node, conditions: &mut f64) { - for operand in list.children().filter(is_operand) { - lua_count_negated(&operand, conditions); - } -} - impl Abc for LuaCode { fn compute<'a>( node: &Node<'a>, @@ -158,10 +140,16 @@ impl Abc for LuaCode { stats.conditions += 1.; } // Fitzpatrick Rule 9 walker: each operand of an `and` / - // `or` chain is one condition (issue #403). + // `or` chain is one condition (issue #403). `a and b or c` + // is a left-nested chain of `binary_expression`s, so an + // operand that is itself a chain is paid by its own + // operator's visit. Lua::And | Lua::Or => { - if let Some(chain) = ancestors.parent(node) { - lua_count_chain_operands(&chain, &mut stats.conditions); + if let Some(chain) = ancestors + .parent(node) + .filter(|p| p.kind_id() == Lua::BinaryExpression) + { + count_field_operands(&chain, lua_count_condition, &mut stats.conditions); } } // An `elseif` is Java's `else if`: the `else` (+1, Rule 5) and @@ -192,12 +180,12 @@ impl Abc for LuaCode { // reports zero. Bare `return` (no values) has no operand. Lua::ReturnStatement => { if let Some(expr_list) = wrapped_operand(node) { - lua_count_negated_operands(&expr_list, &mut stats.conditions); + count_each_operand(&expr_list, lua_count_negated, &mut stats.conditions); } } // `f(not a, not b)` — argument-list walker. Lua::Arguments => { - lua_count_negated_operands(node, &mut stats.conditions); + count_each_operand(node, lua_count_negated, &mut stats.conditions); } _ => {} } diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index d082355a..9e4196b0 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -10,8 +10,8 @@ )] use super::{ - Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, last_operand, - wrapped_operand, + Abc, Stats, count_boolean_slot, count_each_operand, count_negated_operand, is_operand, + last_operand, wrapped_operand, }; use crate::*; @@ -198,9 +198,7 @@ fn perl_count_chain_operands(chain: &Node, conditions: &mut f64) { chain.kind_id().into(), Perl::BinaryExpression | Perl::UnaryExpression ) { - for operand in chain.children().filter(is_operand) { - perl_count_condition(&operand, conditions); - } + count_each_operand(chain, perl_count_condition, conditions); } } @@ -441,9 +439,7 @@ impl Abc for PerlCode { .parent(node) .is_some_and(perl_is_call_argument_parent) => { - for argument in node.children().filter(is_operand) { - perl_count_negated(&argument, &mut stats.conditions); - } + count_each_operand(node, perl_count_negated, &mut stats.conditions); } // `$a ? !$b : !$c`. Unlike the C family, this dispatcher // has no `?`-token arm — the grammar emits the token, but diff --git a/src/metrics/abc/php.rs b/src/metrics/abc/php.rs index 1d62282c..dc1ce2e5 100644 --- a/src/metrics/abc/php.rs +++ b/src/metrics/abc/php.rs @@ -9,7 +9,10 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, wrapped_operand}; +use super::{ + Abc, Stats, count_boolean_slot, count_field_operands, count_negated_operand, is_operand, + wrapped_operand, +}; use crate::*; // One step of the value peel (see `PeelStep`). A parenthesis, a cast @@ -156,17 +159,6 @@ fn php_count_arguments(arguments: &Node, conditions: &mut f64) { } } -// Fitzpatrick Rule 9 (#403): each operand of an `&&` / `||` / `and` / -// `or` / `xor` chain is a boolean slot. `$a && $b || $c` is a -// left-nested chain of `binary_expression`s, so an operand that is -// itself a chain is paid by its own operator's visit. Read by field: a -// comment beside the operator is a named child too, and must not pay. -fn php_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - php_count_slot(chain.child_by_field_name(field), conditions); - } -} - impl Abc for PhpCode { fn compute<'a>( node: &Node<'a>, @@ -235,13 +227,15 @@ impl Abc for PhpCode { // low-precedence keyword forms (`and`, `or`, `xor`) as // distinct tokens inside `binary_expression`; both fire // the walker so `connect() or die();`-style idiom counts - // the same as `connect() || die();`. + // the same as `connect() || die();`. `$a && $b || $c` is a + // left-nested chain of `binary_expression`s, so an operand + // that is itself a chain is paid by its own operator's visit. AMPAMP | PIPEPIPE | And | Or | Xor => { if let Some(chain) = ancestors .parent(node) .filter(|p| p.kind_id() == BinaryExpression) { - php_count_chain_operands(&chain, &mut stats.conditions); + count_field_operands(&chain, php_count_condition, &mut stats.conditions); } } // An `elseif` is Java's `else if` — and PHP's own two-word diff --git a/src/metrics/abc/python.rs b/src/metrics/abc/python.rs index a2da25f3..0b016979 100644 --- a/src/metrics/abc/python.rs +++ b/src/metrics/abc/python.rs @@ -9,7 +9,7 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, peel, wrapped_operand}; +use super::{Abc, Stats, count_boolean_slot, count_field_operands, peel, wrapped_operand}; use crate::*; // Fitzpatrick's ABC rules adapted for Python. @@ -90,19 +90,6 @@ fn python_count_condition(condition: &Node, conditions: &mut f64) { ); } -// Each operand of an `and` / `or` chain is a boolean slot (Fitzpatrick -// Rule 9, #403). tree-sitter-python parses `a and b or c` as a -// left-nested chain of `boolean_operator`s, so an operand that is itself -// a chain is paid by its own operator's visit. Read by field: a comment -// beside the operator is a named child too, and must not pay. -fn python_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - python_count_condition(&operand, conditions); - } - } -} - // Phase-2B (issue #1161): the condition slot of a Python conditional // expression, ` if else `. Before // this, `a if c() else b` reported 1 — the `ConditionalExpression` node @@ -258,12 +245,15 @@ impl Abc for PythonCode { // `or` chain is one condition (issue #403). The `And` / // `Or` keyword tokens live inside a `boolean_operator` // wrapper which the walker iterates as the parent list. + // `a and b or c` is a left-nested chain of + // `boolean_operator`s, so an operand that is itself a chain is + // paid by its own operator's visit. And | Or => { if let Some(chain) = ancestors .parent(node) .filter(|p| p.kind_id() == BooleanOperator) { - python_count_chain_operands(&chain, &mut stats.conditions); + count_field_operands(&chain, python_count_condition, &mut stats.conditions); } } // An `elif` is Java's `else if`: the `else` (+1, Rule 5) and diff --git a/src/metrics/abc/ruby.rs b/src/metrics/abc/ruby.rs index 54f8ec25..05915c34 100644 --- a/src/metrics/abc/ruby.rs +++ b/src/metrics/abc/ruby.rs @@ -9,7 +9,9 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_negated_operand, last_operand}; +use super::{ + Abc, Stats, count_boolean_slot, count_field_operands, count_negated_operand, last_operand, +}; use crate::*; // Ruby ABC rules follow the Fitzpatrick paper's spirit, adapted to @@ -84,22 +86,6 @@ fn ruby_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { } } -// Fitzpatrick Rule 9 walker (#557): each operand of an `&&` / `||` / -// `and` / `or` chain is a boolean slot, scored by `ruby_count_condition`. -// tree-sitter-ruby parses `a && b || c` as a left-nested chain of -// `binary` nodes (aliased `Binary`..`Binary3`, lesson #2), so an operand -// that is itself a chain is paid by its own operator's visit. -// -// The operands are read by field: a comment between operand and operator -// is a named child of the `binary` too, and must not pay. -fn ruby_count_chain_operands(chain: &Node, conditions: &mut f64) { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - ruby_count_condition(&operand, conditions); - } - } -} - // Scores one boolean slot — the predicate of every `if` / `unless` / // `while` / `until` form (block and modifier), a `case … in` guard, a // ternary condition, each pattern of a subject-less `when`, and each @@ -369,13 +355,16 @@ impl Abc for RubyCode { // (cross-language policy, #395); the keyword forms `and` / `or` // get the same treatment as `&&` / `||`. Under error recovery // the token's parent can be an `ERROR` node (`f(&&)`), which - // has no operands to score. + // has no operands to score. tree-sitter-ruby parses + // `a && b || c` as a left-nested chain of `binary` nodes + // (aliased `Binary`..`Binary3`, lesson #2), so an operand that + // is itself a chain is paid by its own operator's visit. AMPAMP | PIPEPIPE | And | Or => { if let Some(chain) = ancestors .parent(node) .filter(|p| matches!(p.kind_id().into(), Binary | Binary2 | Binary3)) { - ruby_count_chain_operands(&chain, &mut stats.conditions); + count_field_operands(&chain, ruby_count_condition, &mut stats.conditions); } } // `a ? !b : !c` — the ternary's own `?` token is already diff --git a/src/metrics/abc/rust.rs b/src/metrics/abc/rust.rs index 6a0eb8ff..4d60b66d 100644 --- a/src/metrics/abc/rust.rs +++ b/src/metrics/abc/rust.rs @@ -10,8 +10,8 @@ )] use super::{ - Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, last_operand, - wrapped_operand, + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, + count_negated_operand, last_operand, wrapped_operand, }; use crate::*; @@ -92,13 +92,6 @@ fn rust_count_slot(slot: Option, conditions: &mut f64) { } } -// Each argument of a call is a negated operand (`m(!a, !b)`). -fn rust_count_arguments(arguments: &Node, conditions: &mut f64) { - for argument in arguments.children().filter(is_operand) { - rust_count_negated(&argument, conditions); - } -} - // The conditions a `match_arm` contributes: the arm itself, plus its // guard. // @@ -151,20 +144,14 @@ fn rust_count_match_arm(node: &Node, conditions: &mut f64) { // 3). Its `let` operands score themselves. fn rust_count_chain_operands(token: &Node, chain: &Node, conditions: &mut f64) { if chain.kind_id() == Rust::BinaryExpression { - for field in ["left", "right"] { - if let Some(operand) = chain.child_by_field_name(field) { - rust_count_condition(&operand, conditions); - } - } + count_field_operands(chain, rust_count_condition, conditions); } else if matches!(chain.kind_id().into(), Rust::LetChain | Rust::LetChain2) && chain .children() .find(|child| child.kind_id() == Rust::AMPAMP) .is_some_and(|first| first.id() == token.id()) { - for operand in chain.children().filter(is_operand) { - rust_count_condition(&operand, conditions); - } + count_each_operand(chain, rust_count_condition, conditions); } } @@ -278,7 +265,8 @@ impl Abc for RustCode { } // Method-argument walker: `m(!a, !b)` contributes one // condition per negated argument. - Arguments => rust_count_arguments(node, &mut stats.conditions), + // Each argument of a call is a negated operand (`m(!a, !b)`). + Arguments => count_each_operand(node, rust_count_negated, &mut stats.conditions), _ => {} } } diff --git a/src/metrics/abc/tcl.rs b/src/metrics/abc/tcl.rs index cf1fcc07..43918847 100644 --- a/src/metrics/abc/tcl.rs +++ b/src/metrics/abc/tcl.rs @@ -9,7 +9,9 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_negated_operand, is_operand, wrapped_operand}; +use super::{ + Abc, Stats, count_boolean_slot, count_each_operand, count_negated_operand, wrapped_operand, +}; use crate::*; // Names of Tcl commands that mutate a variable. Each invocation of @@ -114,9 +116,7 @@ fn tcl_count_condition(condition: &Node, conditions: &mut f64) { // operand field, and inlines the parens of `($a)`. fn tcl_count_chain_operands(chain: &Node, conditions: &mut f64) { if chain.kind_id() == Tcl::BinopExpr { - for operand in chain.children().filter(is_operand) { - tcl_count_condition(&operand, conditions); - } + count_each_operand(chain, tcl_count_condition, conditions); } } From 58991ae86f39bbe82b7e56fe104fef27808b437d Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 09:37:48 -0700 Subject: [PATCH 02/48] fix(abc): score a negated C# call argument tree-sitter-c-sharp wraps every call argument in an `argument` node, and the argument walk handed that wrapper straight to the negation peel, which declines it. So `G(!a)` scored 0 conditions where Java's `g(!a)` and every other C-family language score 1. Read each argument's value as the wrapper's last operand, which skips a named-argument label, a `ref`/`out`/`in` keyword and comments. The primary-constructor base call `class S(bool a) : Base(!a)` returns early as a branch before the walk, while its record spelling reaches it, so score its arguments on the branch path too and keep the two spellings level. Indexer arguments stay unscored, as Java's index is. The unit snapshot for `F(x == y, x < y, !x.Equals(y))` moves from 2 to 3 conditions and is now anchored. No corpus snapshot moves. Fixes #1537 --- .snapshot-anchor-baseline.txt | 2 +- big-code-analysis-book/src/metrics.md | 2 +- src/metrics/abc.rs | 79 ++++++++++++++++++- src/metrics/abc/csharp.rs | 36 +++++++-- ...arp_methods_arguments_with_conditions.snap | 8 +- 5 files changed, 112 insertions(+), 15 deletions(-) diff --git a/.snapshot-anchor-baseline.txt b/.snapshot-anchor-baseline.txt index 70540bf7..2dc4db08 100644 --- a/.snapshot-anchor-baseline.txt +++ b/.snapshot-anchor-baseline.txt @@ -9,7 +9,7 @@ # rather than a census of every scanned file (#1192 made the # scan recursive, which would otherwise have added 126 zeroes). -src/metrics/abc.rs 27 +src/metrics/abc.rs 26 src/metrics/cyclomatic.rs 1 src/metrics/npa.rs 12 src/metrics/npm.rs 23 diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 06e7abe5..cfaddd7b 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -145,7 +145,7 @@ application would over-count. | All languages | `default` / `_` wildcard arm excluded from the condition set | Fitzpatrick's Figure 2 lists `default`, but it falls through unconditionally — counting it would inflate `C` on every `switch` / `match` regardless of body. big-code-analysis omits it for every language (the Rust `_ =>` and Java `default:` arms included). | | Tcl | Chain-operand, bare-truthy and ternary slots wired; argument and `return` slots are not | Each operand of a `&&` / `\|\|` chain inside `expr {…}` counts as one condition, so `if {$a && $b}` reports two. `if` / `elseif` / `while` route their `expr {…}` predicate, so a bare-truthy `if {$a}` reports one and `if {!$a}` likewise — matching C's `if (a)` (#1180). A predicate written as a command substitution is a truthy test of that command's result, so `if {[somecmd]}` also reports one, and the redundant `if {[expr {$a \|\| $b}]}` idiom reports three (the chain's two operands plus the substitution). The argument and `return` slots remain unrouted: a negation reached only through a standalone `expr {…}` command still reports zero. | | iRules | Chain-operand, bare-truthy and ternary slots wired; argument and `return` slots are not | Each operand of a `&&` / `\|\|` / `and` / `or` chain counts as one condition (Rule 9), so `if {!$a && !$b}` reports two. iRules also recognises the word-form string-match comparators (`contains`, `starts_with`, `ends_with`, `equals`, `matches`, …) that Tcl lacks (Tcl's `eq` / `ne` / `in` / `ni` are shared). Bare-truthy and ternary slot routing matches its Tcl sibling exactly (#1180) — see that row for the shared detail, including the argument and `return` slots that remain unrouted. | -| All Phase 2 languages (Java, Groovy, C#, Rust, Go, JavaScript, TypeScript, TSX, Mozjs, PHP, C, C++, Objective-C, Mozcpp, Python, Perl, Lua) | `if (true) {}`, `m(!a, !b)`, `return !x` count their operand(s) | Phase 2B routes `if` / `while` / `do-while` / argument-list / `return` slots through the same walker, so the rule applies uniformly across decision-bearing positions. A bare `return x` continues to report zero — Fitzpatrick treats an identifier in a return slot as a value, not a unary conditional. C# is the one exception to the argument-list half of the rule: its grammar wraps each argument in an `argument` node that the walker does not descend, so `M(!a, !b)` reports zero where every other language in this row reports two. `if (true)` and `return !x` behave as stated. | +| All Phase 2 languages (Java, Groovy, C#, Rust, Go, JavaScript, TypeScript, TSX, Mozjs, PHP, C, C++, Objective-C, Mozcpp, Python, Perl, Lua) | `if (true) {}`, `m(!a, !b)`, `return !x` count their operand(s) | Phase 2B routes `if` / `while` / `do-while` / argument-list / `return` slots through the same walker, so the rule applies uniformly across decision-bearing positions. A bare `return x` continues to report zero — Fitzpatrick treats an identifier in a return slot as a value, not a unary conditional. C# wraps each argument in an `argument` node, so the walker reads the value after any named-argument label or `ref` / `out` / `in` keyword: `M(!a, !b)` and `M(name: !a, !b)` report two, as do the base calls `: base(!a, !b)` and `class S(bool a, bool b) : Base(!a, !b)`. Before #1537 it read the wrapper instead, and every C# argument reported zero. An indexer argument (`o[!a]`) is not a call argument and reports zero in C#, as Java's array index does. | | Ternary slots: Java, Groovy, C#, C, C++, Objective-C, Mozcpp, JavaScript, TypeScript, TSX, Mozjs, PHP, Perl, Ruby, Python, Tcl, iRules | `a ? !b : !c` counts its condition and both branch operands | The same walker also runs over a ternary's three operand slots, so `a ? !b : !c` reports 4 (the `?` plus three unary conditions) rather than 1. Tcl and iRules reach the same 4 without grammar fields: their `ternary_expr` exposes none, and `_expr` inlines `( … )`, so the slots are located relative to the `?` and `:` tokens instead of by index (#1180). Python arrives at the same total by a different route: `not` operands are counted by the `NotOperator` rule wherever they appear, so only the condition slot is routed through the walker and `(not b) if a else (not c)` likewise reports 4. Languages with no ternary (Rust, Go, Kotlin, Lua, Elixir) are unaffected. | | `for` header condition slot: Java, Groovy, C#, C, C++, Objective-C, Mozcpp, JavaScript, TypeScript, TSX, Mozjs, PHP, Go, Perl | `for (; a; )` counts one condition; `for (;;)` counts none | The loop header's condition is a decision-bearing slot like an `if` predicate, so `for (; a; )`, `for (; !a; )` and `for (; (a); )` each report one. A comparison-shaped condition (`i < n`) is already counted by its own operator arm and is not double-counted. Every one of these grammars except Go exposes a `condition` field on `for_statement` (Perl's sits on its C-style `for_statement_1`), so the slot is read by field name rather than by child index; Go's field sits on the nested `for_clause`, and its outer header slot is located structurally (first named child that is neither the body nor a comment). An **empty** condition counts zero in every language: Fitzpatrick's rules count conditional operators and unary conditions that are present, and an omitted test is not a decision. Java and Groovy previously scored `for (;;)` as one vacuously-true condition and no longer do. Tcl, iRules and Bash are command-dispatched grammars whose loop headers carry no boolean-expression slot to route (grammar-dispatch §9) (#1276). | | Ruby | Bare-predicate `if` / `unless` / `while` / `until` (block and modifier forms) count one condition | Idiomatic Ruby favours bare predicates (`if flag`, `x if flag`); counting the condition slot keeps ABC conditions at or above Ruby's cyclomatic decision count (the alignment enforced across the other languages). A comparison (`if a == b`) or `&&` / `\|\|` chain in the predicate is counted by its own operator / walker arm and is not double-counted. Each `&&` / `\|\|` / `and` / `or` operand is scored by the same rule — once, unless its own arm already counts it — so `a && self`, `a && Foo::Bar` and `a && -b` score 2 like `a && b`, wherever the chain is written (#1529). | diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index 9654df42..e01173ff 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -617,7 +617,9 @@ mod tests { #[cfg(any( feature = "c", feature = "cpp", + feature = "csharp", feature = "go", + feature = "java", feature = "javascript", feature = "mozcpp", feature = "mozjs", @@ -3306,10 +3308,85 @@ mod tests { void F(bool a, bool b, bool c) {} }", "foo.cs", - |metric| insta::assert_json_snapshot!(metric.abc), + |metric| { + // expected: `==` +1, `<` +1, and the negated argument + // `!x.Equals(y)` +1 — it scored 0 until #1537, when every + // C# argument reached the peel still in its `argument` + // wrapper. + assert_eq!(metric.abc.conditions_sum(), 3); + insta::assert_json_snapshot!(metric.abc); + }, ); } + // #1537: every C# argument is wrapped in an `argument` node that the + // negation peel declined, so a negated argument scored 0 where every + // other C-family language scores 1. Each row is a C# call next to its + // Java twin, which takes the expected value from outside the fix. + #[cfg(all(feature = "csharp", feature = "java"))] + #[test] + fn csharp_negated_call_argument_matches_java() { + let rows = [ + ("G(!a);", "g(!a);", 1), + ("G((!a));", "g((!a));", 1), + ("G(!a, !b);", "g(!a, !b);", 2), + ("G(a, !b);", "g(a, !b);", 1), + ("new B(!a);", "new B(!a);", 1), + ("G(-a);", "g(-a);", 0), + // A negated comparison pays once, for its `>`; a chain pays + // per operand. Neither argument may pay again as a negation. + ("G(!(a > b));", "g(!(a > b));", 1), + ("G(!a && b);", "g(!a && b);", 2), + ]; + for (csharp, java, expected) in rows { + let cs = abc_conditions( + LANG::Csharp, + &format!("class A {{ void M() {{ {csharp} }} }}"), + ); + let jv = abc_conditions(LANG::Java, &format!("class A {{ void m() {{ {java} }} }}")); + assert_eq!(jv, expected, "Java twin `{java}`"); + assert_eq!(cs, expected, "C# `{csharp}` against Java `{java}`"); + } + } + + // The C#-only spellings around an argument. The value is the + // `argument` wrapper's last operand, so a named-argument label, a + // passing-mode keyword or a comment before it must not hide it. + #[cfg(feature = "csharp")] + #[test] + fn csharp_negated_call_argument_spellings() { + let rows = [ + // A named argument: the value follows `name` and `:`. + ("class A { void M() { G(name: !a); } }", 1), + // A comment between the label and the value sits inside the + // `argument` wrapper, so a positional read of the value lands + // on it. A comment before or after a whole argument is a + // sibling of the wrapper instead and never reaches this read. + ("class A { void M() { G(name: /*c*/ !a); } }", 1), + // `ref` / `out` / `in` precede a variable, never a negation. A + // control only: 0 is also what the pre-#1537 walk scored, so + // this row cannot fail for any defect the fix addresses. + ("class A { void M() { G(ref x, out var y, in z); } }", 0), + // The lambda arm pays for its `!a` body; the argument holding + // the lambda is not a negation and must not pay again. + ("class A { void M() { G(x => !a); } }", 1), + // `: base(!a)` — a `constructor_initializer`'s list reaches the + // walk like any call's. + ("class A : B { A(bool a) : base(!a) {} }", 1), + // The two primary-constructor base calls: the record nests its + // list in a `primary_constructor_base_type`, the class's list + // is the branch node itself and is scored on the branch path. + ("record R(bool a) : Base(!a);", 1), + ("class S(bool a) : Base(!a) {}", 1), + // An indexer's `bracketed_argument_list` is not a call, as + // Java's array index is not. + ("class A { void M() { var q = o[!a]; } }", 0), + ]; + for (src, expected) in rows { + assert_eq!(abc_conditions(LANG::Csharp, src), expected, "{src}"); + } + } + #[cfg(feature = "csharp")] #[test] fn csharp_if_single_conditions() { diff --git a/src/metrics/abc/csharp.rs b/src/metrics/abc/csharp.rs index 66b1b7b9..5ceee2fb 100644 --- a/src/metrics/abc/csharp.rs +++ b/src/metrics/abc/csharp.rs @@ -166,7 +166,21 @@ fn csharp_count_negated_slot(slot: Option, conditions: &mut f64) { // Each argument of a call is a negated operand. fn csharp_count_arguments(arguments: &Node, conditions: &mut f64) { - count_each_operand(arguments, csharp_count_negated, conditions); + count_each_operand(arguments, csharp_count_argument, conditions); +} + +// tree-sitter-c-sharp wraps every argument in an `argument` node — +// `seq(optional(seq(field('name', …), ':')), optional('ref' | 'out' | +// 'in'), expression | declaration_expression)` — so the value is its +// last operand, after any named-argument label (`name: !a`) or +// passing-mode keyword. Handing the wrapper itself to the peel scored +// every argument zero, `G(!a)` included (#1537). The value is located +// by role rather than by index because both prefixes are optional +// (grammar-dispatch §3). +fn csharp_count_argument(argument: &Node, conditions: &mut f64) { + if let Some(value) = last_operand(argument) { + csharp_count_negated(&value, conditions); + } } // ABC token-level helpers for C#. Mirror of Java's helper layout with @@ -629,13 +643,11 @@ fn csharp_walk_for_conditions<'a>( count_field_operands(&chain, csharp_count_condition, conds); } } - // `compute` returns as soon as `csharp_count_token_branch` fires, - // so since #1406 an `argument_list` under a `base_list` no longer - // reaches this arm. Measured harmless: the arm is dead for *every* - // argument list, because an `argument_list`'s children are - // `argument` wrappers that `csharp_wrapper_operand` declines — - // `Helper(!b)` and `Helper((b))` both score zero conditions today. - // Repairing it means revisiting that exclusion, not just this arm. + // Every `argument_list` but the one a `base_list` holds directly: + // that one is itself the branch, so `compute` scores its + // arguments on the branch path instead. A `bracketed_argument_list` + // (an indexer, `o[!a]`) is a distinct kind and is not walked, as + // Java's array index is not. ArgumentList => csharp_count_arguments(node, conds), // tree-sitter-c-sharp spells the parens of `if` / `while` / // `do … while` as anonymous tokens, NOT a wrapping @@ -743,6 +755,14 @@ impl Abc for CsharpCode { return; } if csharp_count_token_branch(node, ancestors, stats) { + // `class S(bool a) : Base(!a)`: the `argument_list` under the + // `base_list` is the branch node itself, so the walk below + // never sees it. The record spelling nests its list inside a + // `primary_constructor_base_type` and reaches the walk, so + // without this the two spellings of one base call disagree. + if node.kind_id() == Csharp::ArgumentList { + csharp_count_arguments(node, &mut stats.conditions); + } return; } if csharp_count_token_condition(node, ancestors, stats) { diff --git a/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__csharp_methods_arguments_with_conditions.snap b/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__csharp_methods_arguments_with_conditions.snap index ac6d3271..00a20ea8 100644 --- a/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__csharp_methods_arguments_with_conditions.snap +++ b/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__csharp_methods_arguments_with_conditions.snap @@ -5,16 +5,16 @@ expression: metric.abc { "assignments": 0, "branches": 2, - "conditions": 2, - "magnitude": 2.8284271247461903, + "conditions": 3, + "magnitude": 3.605551275463989, "value": 0.0, "assignments_average": 0.0, "branches_average": 0.5, - "conditions_average": 0.5, + "conditions_average": 0.75, "assignments_min": 0, "assignments_max": 0, "branches_min": 0, "branches_max": 2, "conditions_min": 0, - "conditions_max": 2 + "conditions_max": 3 } From 4822718dc258ce8a1997da6c04c1be3369bce311 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 09:51:58 -0700 Subject: [PATCH 03/48] test(abc): pin C-family truthy literal slots #1526 closed the C-family non-numeric literal gap without naming a kind: every boolean slot now pays unless its predicate already scores. Nothing pinned that answer for the literal kinds, so a kind wrongly added to cpp_condition_scores_itself would drop a slot silently. Add C, C++, Mozcpp and Objective-C rows to literal_bool_operands, one spelling per non-numeric literal kind, each anchored to its kind id and measured against the identifier twin in both the `&&` chain slot and the `if` slot: string_literal, concatenated_string, true, false and null; C++ raw_string_literal and user_defined_literal; Objective-C @"s", array/dictionary literals, at_expression, @selector and @encode. A negated literal joins the shared C-family slot rows. Fixes #1469 --- src/metrics/abc.rs | 100 ++++++++++++++++++++++++++++- src/metrics/abc/slot_pays_tests.rs | 3 + 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index e01173ff..9968fc05 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -18110,8 +18110,11 @@ mod perl_statement_modifier_parity { /// C#, Java, Kotlin, Rust and Go name no literal kind at all and stay /// that way: a bare literal in a boolean slot is a compile error there, /// so there is nothing to count. The C family is integer-truthy, and -/// its slots have paid for a `string_literal` since #1526 made every -/// slot pay for an occupant no other arm counts. +/// #1462 deferred it for its corpus exposure. #1526 then closed its gap +/// without naming a kind, by making every slot pay for an occupant no +/// other arm counts; its rows here (#1469) pin that answer, because a +/// kind `cpp_condition_scores_itself` wrongly claimed would drop a slot +/// exactly as the missing terminal kind once did. /// /// Three things each row pins that a conditions comparison alone /// cannot: @@ -18149,7 +18152,11 @@ mod perl_statement_modifier_parity { feature = "groovy", feature = "perl", feature = "ruby", - feature = "elixir" + feature = "elixir", + feature = "c", + feature = "cpp", + feature = "mozcpp", + feature = "objc" ))] mod literal_bool_operands { use crate::test_support::{assert_fixture_spells, metrics_verbatim}; @@ -18197,6 +18204,29 @@ mod literal_bool_operands { ("function f() {\n if ({}) { return 1; }\n}\n", 1, 3), ]; + /// The five non-numeric literal kinds every C-family grammar shares, + /// then the grammar's own. The string and null spellings are passed + /// in because Objective-C writes its strings `@"s"` and C++ its null + /// `nullptr`; each still parses to the shared `string_literal` / + /// `null` kind, so the row stays one spelling per kind. + macro_rules! c_family_literals { + ($Lang:ident, $string:literal, $null:literal $(, $extra:expr)* $(,)?) => { + &[ + ($string, crate::$Lang::StringLiteral as u16), + ("\"a\" \"b\"", crate::$Lang::ConcatenatedString as u16), + ("true", crate::$Lang::True as u16), + ("false", crate::$Lang::False as u16), + ($null, crate::$Lang::Null as u16), + $($extra,)* + ] + }; + } + + const C_FAMILY_SLOTS: [Slot; 2] = [ + ("int f(int a) {\n return a && {};\n}\n", 2, 3), + ("int f() {\n if ({}) { return 1; }\n}\n", 1, 3), + ]; + fn conditions(lang: LANG, source: &str) -> u64 { metrics_verbatim(lang, source.as_bytes(), MetricsOptions::default()) .abc @@ -18243,6 +18273,14 @@ mod literal_bool_operands { LANG::Ruby => assert_fixture_spells::(source, "f.rb", kinds), #[cfg(feature = "elixir")] LANG::Elixir => assert_fixture_spells::(source, "f.ex", kinds), + #[cfg(feature = "c")] + LANG::C => assert_fixture_spells::(source, "f.c", kinds), + #[cfg(feature = "cpp")] + LANG::Cpp => assert_fixture_spells::(source, "f.cpp", kinds), + #[cfg(feature = "mozcpp")] + LANG::Mozcpp => assert_fixture_spells::(source, "f.cpp", kinds), + #[cfg(feature = "objc")] + LANG::Objc => assert_fixture_spells::(source, "f.m", kinds), other => panic!("{other:?} has a case row but no parser arm"), } } @@ -18291,6 +18329,17 @@ mod literal_bool_operands { /// `quoted_atom` — a separate production from `atom`, so /// `:"q a"` scored zero while `:atom` scored one — and the four /// collection literals `list` / `tuple` / `map` / `bitstring`. + /// - C family: `string_literal` (one kind for `"s"`, `L"s"`, `u8"s"` + /// and Objective-C's `@"s"`), `concatenated_string`, `true`, + /// `false` and `null` (`NULL`, and C++'s `nullptr`, which the + /// grammar wraps in a `null` node). C++ adds `raw_string_literal` + /// and `user_defined_literal` (`"s"_x`, and `1.0_km` around a + /// number); Objective-C adds its boxed forms — `array_literal`, + /// `dictionary_literal`, `at_expression` (`@1`, `@YES`, `@(b)`, + /// `@protocol(P)`), `selector_expression` and `encode_expression`. + /// `char_literal` is numeric and pinned in the sibling module. + /// Objective-C's `nil`, `YES` and `NO` parse as plain identifiers, + /// so they need no row of their own. /// /// Shapes that measured short and are deliberately absent, none of /// them a literal: JavaScript's `this` and Groovy's @@ -18445,6 +18494,51 @@ mod literal_bool_operands { ], 8, ), + LANG::C => ( + C_FAMILY_SLOTS, + "b", + c_family_literals!(C, "\"s\"", "NULL"), + 5, + ), + LANG::Cpp => ( + C_FAMILY_SLOTS, + "b", + c_family_literals!( + Cpp, + "\"s\"", + "nullptr", + ("R\"(x)\"", crate::Cpp::RawStringLiteral as u16), + ("\"s\"_x", crate::Cpp::UserDefinedLiteral as u16), + ), + 7, + ), + LANG::Mozcpp => ( + C_FAMILY_SLOTS, + "b", + c_family_literals!( + Mozcpp, + "\"s\"", + "nullptr", + ("R\"(x)\"", crate::Mozcpp::RawStringLiteral as u16), + ("\"s\"_x", crate::Mozcpp::UserDefinedLiteral as u16), + ), + 7, + ), + LANG::Objc => ( + C_FAMILY_SLOTS, + "b", + c_family_literals!( + Objc, + "@\"s\"", + "NULL", + ("@[]", crate::Objc::ArrayLiteral as u16), + ("@{}", crate::Objc::DictionaryLiteral as u16), + ("@1", crate::Objc::AtExpression as u16), + ("@selector(x)", crate::Objc::SelectorExpression as u16), + ("@encode(int)", crate::Objc::EncodeExpression as u16), + ), + 10, + ), _ => return None, }) } diff --git a/src/metrics/abc/slot_pays_tests.rs b/src/metrics/abc/slot_pays_tests.rs index d7e38095..e2648bb9 100644 --- a/src/metrics/abc/slot_pays_tests.rs +++ b/src/metrics/abc/slot_pays_tests.rs @@ -259,6 +259,8 @@ const C_FAMILY_IF_ROWS: &[(&str, u64, u64)] = &[ ("!(b > 1)", 1, 1), ("b > 1", 1, 1), ("!b", 1, 1), + // A negated literal is a negation like any other (#1469). + ("!\"s\"", 1, 1), // A ternary predicate is counted by its own `?` and pays its // condition slot; the `if` slot must not pay a third time. ("a ? b : y", 2, 2), @@ -301,6 +303,7 @@ fn assert_c_family_slots(lang: LANG) { ("-b", 0, 0), ("!b", 1, 0), ("!*p", 1, 0), + ("!\"s\"", 1, 0), ("!(b > 1)", 1, 0), ], ); From 16f25af7fb3c25082c3f59f4f3c60bce54e1b0e0 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 09:57:49 -0700 Subject: [PATCH 04/48] test(abc): pin the #1475 boolean-slot constructs #1475 recorded constructs that measured one condition short of an identifier in a boolean slot. #1526 made every slot pay unless its predicate already scores, which fixed all of them, but no test pinned the result. Pin each against the identifier control in both an `if` slot and an `&&` chain-operand slot: C# default/checked/unchecked, an assignment of a call and an object creation; Groovy `new Foo()` and `a*.b`; Perl s///, tr///, y///, an anonymous sub, @a, @$b, %h and %$h; Ruby both lambda spellings and a range; Elixir `fn` and `&f/1`. JS `!this` gains a call-argument row. Pin Groovy::Identifier2 as unreachable: it is the hidden `_gstring_dollar_path` token, emitted under the public `identifier` symbol, so the Halstead operand arm needs no second alias. Whether Perl s/// and tr/// count outside a slot stays undecided; the bound `$x =~ s///` and bare `s///` disagree there, tracked in #1540. Fixes #1475 --- src/metrics/abc.rs | 19 +++-- src/metrics/abc/perl.rs | 6 +- src/metrics/abc/slot_pays_tests.rs | 130 +++++++++++++++++++++++++++++ src/metrics/halstead.rs | 31 +++++++ 4 files changed, 175 insertions(+), 11 deletions(-) diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index 9968fc05..22946584 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -14451,7 +14451,7 @@ end // `=~` / `!~` — the operator token scores, the pattern under it does // not — and `ctrl_out` is the `==` control the rest level with. // `s///`, `tr///` and `qr//` are not matches and stay at 0; whether - // the first two should score is #1475. + // the first two should score by use is #1540. #[cfg(feature = "perl")] #[test] fn perl_bare_match_is_a_condition_by_use() { @@ -18114,7 +18114,9 @@ mod perl_statement_modifier_parity { /// without naming a kind, by making every slot pay for an occupant no /// other arm counts; its rows here (#1469) pin that answer, because a /// kind `cpp_condition_scores_itself` wrongly claimed would drop a slot -/// exactly as the missing terminal kind once did. +/// exactly as the missing terminal kind once did. Its `true`, `false` +/// and `null` rows are the exception to "measured short": the former +/// set already named them, and they stay so the pin covers every kind. /// /// Three things each row pins that a conditions comparison alone /// cannot: @@ -18341,15 +18343,14 @@ mod literal_bool_operands { /// Objective-C's `nil`, `YES` and `NO` parse as plain identifiers, /// so they need no row of their own. /// - /// Shapes that measured short and are deliberately absent, none of - /// them a literal: JavaScript's `this` and Groovy's + /// Shapes that measured short and are not rows here, none of them a + /// literal: JavaScript's `this` and Groovy's /// `object_creation_expression` (recorded in #1462); Perl's /// `s///` and `tr///` (operations on `$_` evaluating to a count), - /// `anonymous_function` and `array_dereference`; and Ruby's - /// `lambda`. Elixir's `anonymous_function` and `&f/1` capture were - /// on this list until #1527 moved its slots off a terminal-kind set, - /// after which every operand scores unless an operator already - /// counts it. + /// `anonymous_function` and `array_dereference`; Ruby's `lambda`; + /// and Elixir's `anonymous_function` and `&f/1` capture. Every one + /// pays its slot since #1526 (Elixir's since #1527), and + /// `slot_pays_tests` pins them against an identifier control (#1475). fn cases(lang: LANG) -> Option { Some(match lang { LANG::Javascript => (JS_SLOTS, "b", js_literals!(Javascript, String2), 7), diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index 9e4196b0..3c0029e2 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -396,8 +396,10 @@ impl Abc for PerlCode { // written (#1467) — `my $r = /^#/` levels with // `my $r = ($x =~ /^#/)`. `perl_condition_scores_itself` // lists it, or every slot would score it a second time. - // `s///` and `tr///` stay out: they edit - // `$_` and yield a count, a policy question left to #1475. + // `s///` and `tr///` stay out: they edit `$_` and yield a + // count. A slot still pays for either (#1475), but by use the + // bound `$x =~ s///` scores through `=~` and the bare form + // does not — the disagreement #1540 tracks. P::PatternMatcher | P::PatternMatcherM if !perl_pattern_is_bound_or_delimiter(node, code, ancestors) => { diff --git a/src/metrics/abc/slot_pays_tests.rs b/src/metrics/abc/slot_pays_tests.rs index e2648bb9..6b6b3644 100644 --- a/src/metrics/abc/slot_pays_tests.rs +++ b/src/metrics/abc/slot_pays_tests.rs @@ -41,6 +41,32 @@ fn assert_rows(lang: LANG, template: &str, rows: &[(&str, u64, u64)]) { } } +/// Asserts each of #1475's constructs pays its `if` slot and its `&&` +/// chain-operand slot exactly as the identifier `control` does: one +/// condition and one decision in the `if`, two of each in the chain. +/// The control row runs first, so a template that stopped measuring +/// fails there rather than in every construct row. +#[cfg(any( + feature = "csharp", + feature = "groovy", + feature = "perl", + feature = "ruby", + feature = "elixir" +))] +#[track_caller] +fn assert_level_with_control( + lang: LANG, + [if_slot, chain_slot]: [&str; 2], + control: &str, + constructs: &[&str], +) { + let predicates = || std::iter::once(control).chain(constructs.iter().copied()); + let in_if: Vec<_> = predicates().map(|p| (p, 1, 1)).collect(); + assert_rows(lang, if_slot, &in_if); + let in_chain: Vec<_> = predicates().map(|p| (p, 2, 2)).collect(); + assert_rows(lang, chain_slot, &in_chain); +} + #[test] #[cfg(feature = "python")] fn python_slots_pay_for_any_predicate() { @@ -180,6 +206,13 @@ fn assert_js_family_slots(lang: LANG) { ("!(b > 1)", 1, 0), ], ); + // A call argument is no slot either; the negation still counts, + // and `this` scores like an identifier there too (#1475). + assert_rows( + lang, + &function.replace(SLOT, "g(PRED);"), + &[("b", 0, 0), ("!b", 1, 0), ("!this", 1, 0)], + ); } #[test] @@ -695,6 +728,31 @@ fn csharp_slots_pay_for_any_predicate() { ); } +/// The C# spellings #1475 measured at zero before #1526 made every slot +/// pay: `default(bool)`, the `checked` wrapper (and its `unchecked` +/// twin), an assignment whose value is a call, and an object creation +/// converted to `bool` by a user-defined operator. +#[test] +#[cfg(feature = "csharp")] +fn csharp_1475_constructs_pay_their_slot() { + let function = "class C { bool y; bool f(bool a, bool b) { PRED return a; } }"; + assert_level_with_control( + LANG::Csharp, + [ + &function.replace(SLOT, "if (PRED) { g(); }"), + &function.replace(SLOT, "if (a && (PRED)) { g(); }"), + ], + "b", + &[ + "default(bool)", + "checked(b)", + "unchecked(b)", + "y = C()", + "new Wrapper()", + ], + ); +} + /// PHP is truthy-valued, so every predicate is a decision. `??` is no /// condition token in PHP, so a slot holding one pays for it. #[test] @@ -890,6 +948,24 @@ fn groovy_slots_pay_for_any_predicate() { ); } +/// #1475's Groovy rows: an object creation, which JS and PHP already +/// counted, and a spread-dot navigation, whose list result is tested +/// for emptiness. +#[test] +#[cfg(feature = "groovy")] +fn groovy_1475_constructs_pay_their_slot() { + let function = "class C { def f(a, b) { PRED; return a } }\n"; + assert_level_with_control( + LANG::Groovy, + [ + &function.replace(SLOT, "if (PRED) { g() }"), + &function.replace(SLOT, "if (a && (PRED)) { g() }"), + ], + "b", + &["new Foo()", "a*.b"], + ); +} + /// Perl is truthy-valued, so every predicate is a decision. #[test] #[cfg(feature = "perl")] @@ -975,6 +1051,60 @@ fn perl_elsif_is_an_else_and_a_slot() { ); } +/// #1475's Perl rows: the substitution and transliteration operators +/// (all three spellings), an anonymous sub, and the array and hash +/// variables in both their named and dereferencing forms. `@$b` was +/// the case #1475 called the clearest defect, since `@a` scored and it +/// did not. Each pays a slot; whether `s///` and `tr///` count outside +/// one is a separate question (#1540), which this test does not settle. +#[test] +#[cfg(feature = "perl")] +fn perl_1475_constructs_pay_their_slot() { + let function = "sub f { my ($a, $b, $h) = @_; my @a; my %h; PRED }\n"; + assert_level_with_control( + LANG::Perl, + [ + &function.replace(SLOT, "if (PRED) { g(); }"), + &function.replace(SLOT, "if ($a && (PRED)) { g(); }"), + ], + "$b", + &[ + "s/x/y/", "tr/a/b/", "y/a/b/", "sub {1}", "@a", "@$b", "%h", "%$h", + ], + ); +} + +/// #1475's Ruby rows: both lambda spellings and a range. +#[test] +#[cfg(feature = "ruby")] +fn ruby_1475_constructs_pay_their_slot() { + assert_level_with_control( + LANG::Ruby, + [ + "def f(a, b)\n if PRED\n g\n end\nend\n", + "def f(a, b)\n if a && (PRED)\n g\n end\nend\n", + ], + "b", + &["lambda {}", "-> {}", "(1..2)"], + ); +} + +/// #1475's Elixir rows: an anonymous function and a function capture. +#[test] +#[cfg(feature = "elixir")] +fn elixir_1475_constructs_pay_their_slot() { + let function = "defmodule M do\n def f(a, b) do\n PRED\n end\nend\n"; + assert_level_with_control( + LANG::Elixir, + [ + &function.replace(SLOT, "if PRED do\n g()\n end"), + &function.replace(SLOT, "if a && (PRED) do\n g()\n end"), + ], + "b", + &["fn -> 1 end", "&f/1"], + ); +} + /// Tcl's `expr` tests any number for non-zero, so every predicate is a /// decision. iRules is its dialect and shares every row. #[cfg(any(feature = "tcl", feature = "irules"))] diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index c0a49175..e2c2c121 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -3020,6 +3020,37 @@ mod tests { assert_ops_operands::(src, "foo.groovy", 3, vec!["greet", "name", "$name"]); } + // `Groovy::Identifier2` (141) is the grammar's external token + // `_gstring_dollar_path` — the `$a.b` path of a dollar interpolation — + // whose public symbol is `identifier`, so the parser reports it as + // the unsuffixed `Identifier` and never as 141 (#1475). That is why + // the operand arm lists `Identifier` but not `Identifier2`; if a + // grammar bump starts emitting 141, the dotted path drops out of N2 + // with no error. + // + // expected: operands `greet`, `name` (param), `$name.b` (the whole + // path, one node) → n2 = 3, N2 = 3. + #[cfg(feature = "groovy")] + #[test] + fn groovy_gstring_dollar_path_is_the_public_identifier() { + let src = "def greet(name) {\n return \"Hi $name.b\"\n}\n"; + let parser = GroovyParser::new(src.as_bytes().to_vec(), &PathBuf::from("foo.groovy"), None); + assert!( + ast_has_kind_id(&parser, Groovy::GstringDollarInterpolation as u16), + "the fixture no longer spells a dollar interpolation", + ); + assert!( + !ast_has_kind_id(&parser, Groovy::Identifier2 as u16), + "the hidden `_gstring_dollar_path` token is now emitted as \ + `Identifier2`; add it to the Groovy operand arm", + ); + check_metrics::(src, "foo.groovy", |metric| { + assert_eq!(metric.halstead.unique_operands(), 3); + assert_eq!(metric.halstead.total_operands(), 3); + }); + assert_ops_operands::(src, "foo.groovy", 3, vec!["greet", "name", "$name.b"]); + } + #[cfg(feature = "groovy")] #[test] fn groovy_plain_string_still_operand() { From 9be55c3ef1fc13438bb629cab488d3aa1c31fce8 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:22:42 -0700 Subject: [PATCH 05/48] fix(abc/perl): skip operands a comparison misparses tree-sitter-perl 1.1.2 lets a chaining comparison (< > <= >= lt gt le ge == != eq ne) swallow every lower-precedence operator to its right, so `$a > $b ? 1 : 2` parses as `$a > ($b ? 1 : 2)` and `$a > $b && $c` as `$a > ($b && $c)`. The ternary condition slot and the chain walker then paid for `$b`, scoring one condition above the JavaScript twin per misparsed comparison. Detect the shape by climbing the ternary-condition / chain-first- operand spine to a node that directly follows a chaining comparison token, and skip that leading slot: the real occupant is the comparison, which scores itself. Marked FIXME(#1273 upstream). Cyclomatic and cognitive are unaffected. Fixes #1273 --- big-code-analysis-book/src/metrics.md | 1 + src/metrics/abc.rs | 86 ++++++++++++++++++++ src/metrics/abc/perl.rs | 112 ++++++++++++++++++++++++-- 3 files changed, 192 insertions(+), 7 deletions(-) diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index cfaddd7b..e5a6b7d2 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -160,6 +160,7 @@ application would over-count. | C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these `<` / `>` as an operator, and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | | C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | +| Perl | A comparison's right operand is not a condition, although the grammar parses it as one | tree-sitter-perl 1.1.2 lets a chaining comparison (`<`, `>`, `<=`, `>=`, `lt`, `gt`, `le`, `ge`, `==`, `!=`, `eq`, `ne`) take everything of lower precedence to its right as its operand, so `$a > $b ? 1 : 2` parses as `$a > ($b ? 1 : 2)` and `$a > $b && $c` as `$a > ($b && $c)`. The ternary's condition slot and the chain's first operand then held `$b`, which scored as a condition of its own: one above the JavaScript twin for each misparsed comparison. The walker recognises the shape and leaves that operand to the comparison, so both spellings score what the parenthesised `($a > $b) ? 1 : 2` and `($a > $b) && $c` do. An explicit `$a > ($b ? 1 : 2)` is unaffected. `<=>`, `cmp`, `=~` and `!~` do not chain and parse correctly. The workaround goes when the grammar is fixed upstream (#1273). | #### Where a constructor call lands {#abc-constructor-attribution} diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index 22946584..3499c978 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -14130,6 +14130,92 @@ end ); } + // Issue #1273. tree-sitter-perl 1.1.2 parses `$a > $b ? 1 : 2` as + // `$a > ($b ? 1 : 2)`, and the ternary's condition slot then paid + // for `$b`: 3 against JavaScript's 2 for `a > b ? 1 : 2`. Every + // chaining comparison does it; each row must also equal its + // parenthesised spelling, which parses correctly and so takes its + // value from outside the fix. + #[cfg(feature = "perl")] + #[test] + fn perl_comparison_conditioned_ternary_counts_the_comparison_once() { + let operators = [ + "<", ">", "<=", ">=", "lt", "gt", "le", "ge", "==", "!=", "eq", "ne", + ]; + for op in operators { + let bare = format!("sub f {{ my $m = $a {op} $b ? 1 : 2; }}\n"); + let grouped = format!("sub f {{ my $m = ($a {op} $b) ? 1 : 2; }}\n"); + // The ternary (1) and the comparison token (1). + assert_eq!(abc_conditions(LANG::Perl, &bare), 2, "`{op}`"); + assert_eq!(abc_conditions(LANG::Perl, &grouped), 2, "`({op})`"); + } + } + + // Issue #1273, the general shape: a chaining comparison swallows + // every lower-precedence operator to its right, so a logical chain + // misparses too — `$a > $b && $c` is `$a > ($b && $c)`, and the + // chain walker paid for `$b`. Expected values are the JavaScript + // twins (`a > b && c ? 1 : 2` = 3, …) or, where JavaScript has no + // spelling, the parenthesised Perl one. + #[cfg(feature = "perl")] + #[test] + fn perl_misparsed_comparison_operand_is_not_a_condition() { + let cases = [ + // `<` `<` `?`; `($a < $b < $c) ? 1 : 2` is 3. + ("my $m = $a < $b < $c ? 1 : 2;", 3), + // `>` `?` `>` `?`, as `a > b ? (c > d ? 1 : 2) : 3`. + ("my $m = $a > $b ? $c > $d ? 1 : 2 : 3;", 4), + // Two ternaries and two comparisons, as the JS twin. + ("if ($a > $b ? $c : $d) { } return $a == $b ? 1 : 0;", 4), + // The chain is the ternary's condition: `>` `?` `$c`. + ("my $m = $a > $b && $c ? 1 : 2;", 3), + // The ternary hangs off the chain's right comparison. + ("my $m = $x > 0 && $y > 0 ? 1 : 2;", 3), + ("my $m = $x && $a > $b ? 1 : 2;", 3), + ("if ($a > $b && $c) { }", 2), + ("if ($a eq $b && $c eq $d) { }", 2), + // `&&` is the first operand of `||`: the climb crosses it. + ("if ($a > $b && $c || $d) { }", 3), + ("if ($a > $b && $c && $d) { }", 3), + // `and` is a two-operand `unary_expression`, which the + // climb crosses too: `(($a > $b) && $c) and $d` is 3. + ("if ($a > $b and $c) { }", 2), + ("if ($a > $b && $c and $d) { }", 3), + // A comment between the operator and the ternary is an + // `extra` sibling, not the operator. + ("my $m = $a > # c\n $b ? 1 : 2;", 2), + ]; + for (body, expected) in cases { + let src = format!("sub f {{ {body} }}\n"); + assert_eq!(abc_conditions(LANG::Perl, &src), expected, "`{body}`"); + } + } + + // Issue #1273: shapes the workaround must leave alone. The explicit + // `$a > ($b ? 1 : 2)` puts an `array` between the comparison and + // the ternary, so `$b` really is a condition: 3, as + // `a > (b ? 1 : 2)` in JavaScript. `<=>`, `cmp` and `=~` do not + // chain and parse correctly; those rows pin that grammar fact, not + // the workaround's operator list — adding `<=>` to the list moves + // nothing, since no tree ever puts a ternary straight after it. + #[cfg(feature = "perl")] + #[test] + fn perl_correctly_parsed_ternaries_keep_their_condition_slot() { + let cases = [ + ("my $m = ($a > $b) ? 1 : 2;", 2), + ("my $m = $a <=> $b ? 1 : 2;", 2), + ("my $m = $a cmp $b ? 1 : 2;", 2), + ("my $m = $a =~ /x/ ? 1 : 2;", 2), + ("my $m = $a > ($b ? 1 : 2);", 3), + ("my $m = $a ? !$b : !$c;", 4), + ("my $m = $a && $b ? 1 : 2;", 3), + ]; + for (body, expected) in cases { + let src = format!("sub f {{ {body} }}\n"); + assert_eq!(abc_conditions(LANG::Perl, &src), expected, "`{body}`"); + } + } + #[cfg(feature = "perl")] #[test] fn perl_elsif_and_else_count_conditions() { diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index 3c0029e2..69dba7c6 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -168,6 +168,88 @@ fn perl_count_slot(slot: Option, conditions: &mut f64) { } } +// FIXME(#1273 upstream): tree-sitter-perl 1.1.2 lets a chaining +// comparison (`< > <= >= lt gt le ge == != eq ne`) swallow everything +// of lower precedence to its right, so `$a > $b && $c` parses as +// `$a > ($b && $c)` and `$a > $b ? 1 : 2` as `$a > ($b ? 1 : 2)`. The +// operand those trees hand to a boolean slot — `$b` — is really the +// comparison's right operand, and the real slot occupant is the +// comparison, which scores itself. Remove these helpers, and the two +// call sites' `absorbed` flags, once the pin carries the upstream +// `prec` fix. +// +// Whether the leading boolean slot of `node` — a ternary's condition or +// a logical chain's first operand — belongs to such a comparison. The +// misparse nests the stolen operand down a spine of exactly these two +// slots (`$a > $b && $c ? …` is `$a > (($b && $c) ? …)`), so the climb +// follows that spine to its root and asks whether the root directly +// follows a chaining operator. An explicit `$a > ($b ? 1 : 2)` puts an +// `array` between the two and is left alone. The token check alone +// suffices: those operators occur only in `binary_expression`, apart +// from the readline `` brackets, whose interior is never an +// expression. +fn perl_misparse_absorbs_leading_operand<'a>( + node: &Node<'a>, + mut ancestors: impl Iterator>, +) -> bool { + use Perl as P; + + let mut child = *node; + ancestors + .find(|parent| { + let leading = match parent.kind_id().into() { + P::TernaryExpression => parent.child_by_field_name("condition"), + _ if perl_is_logical_chain(parent) => wrapped_operand(parent), + _ => None, + }; + let on_spine = leading.is_some_and(|lead| lead.id() == child.id()); + if on_spine { + child = *parent; + } + !on_spine + }) + .is_some_and(|parent| { + parent + .children() + .take_while(|sibling| sibling.id() != child.id()) + .filter(|sibling| !sibling.as_tree_sitter().is_extra()) + .last() + .is_some_and(|operator| { + matches!( + operator.kind_id().into(), + P::LT + | P::GT + | P::LTEQ + | P::GTEQ + | P::Lt + | P::Gt + | P::Le + | P::Ge + | P::EQEQ + | P::BANGEQ + | P::Eq + | P::Ne + ) + }) + }) +} + +// A `&&` / `||` / `//` / `and` / `or` / `xor` chain. tree-sitter-perl +// spells `and` as a two-operand `unary_expression`. +fn perl_is_logical_chain(node: &Node) -> bool { + use Perl as P; + + matches!( + node.kind_id().into(), + P::BinaryExpression | P::UnaryExpression + ) && node.children().any(|token| { + matches!( + token.kind_id().into(), + P::AMPAMP | P::PIPEPIPE | P::SLASHSLASH | P::And | P::Or | P::Xor + ) + }) +} + // Phase-2B (issues #403 / #1102): a ternary's condition is a boolean // slot and each branch a negated operand, exactly as `java_walk_ternary` // counts them. Without this Perl scored `$a ? !$b : !$c` as 1 (the @@ -177,8 +259,12 @@ fn perl_count_slot(slot: Option, conditions: &mut f64) { // tree-sitter-perl names the branches `true` / `false` rather than the // C-family `consequence` / `alternative`, and all three slots are // mandatory — Perl has no short-ternary elision. -fn perl_walk_ternary(node: &Node, conditions: &mut f64) { - perl_count_slot(node.child_by_field_name("condition"), conditions); +// +// `condition_absorbed`: see `perl_misparse_absorbs_leading_operand`. +fn perl_walk_ternary(node: &Node, condition_absorbed: bool, conditions: &mut f64) { + if !condition_absorbed { + perl_count_slot(node.child_by_field_name("condition"), conditions); + } for field in ["true", "false"] { if let Some(branch) = node.child_by_field_name(field) { perl_count_negated(&branch, conditions); @@ -193,12 +279,20 @@ fn perl_walk_ternary(node: &Node, conditions: &mut f64) { // `extra`: tree-sitter-perl names no operand field on a comparison or on // the two-operand `unary_expression` it parses `$a and $b` as, which the // field-less walk is what scores at all — it scored 0. -fn perl_count_chain_operands(chain: &Node, conditions: &mut f64) { +// +// `leading_absorbed`: see `perl_misparse_absorbs_leading_operand`. +fn perl_count_chain_operands(chain: &Node, leading_absorbed: bool, conditions: &mut f64) { if matches!( chain.kind_id().into(), Perl::BinaryExpression | Perl::UnaryExpression ) { - count_each_operand(chain, perl_count_condition, conditions); + for operand in chain + .children() + .filter(is_operand) + .skip(usize::from(leading_absorbed)) + { + perl_count_condition(&operand, conditions); + } } } @@ -410,8 +504,10 @@ impl Abc for PerlCode { // condition (issue #403). Covers `&&`, `||`, `//`, // `and`, `or`, `xor`. P::AMPAMP | P::PIPEPIPE | P::SLASHSLASH | P::And | P::Or | P::Xor => { - if let Some(chain) = ancestors.parent(node) { - perl_count_chain_operands(&chain, &mut stats.conditions); + let mut above = ancestors.iter(node).map(|(ancestor, _)| ancestor); + if let Some(chain) = above.next() { + let absorbed = perl_misparse_absorbs_leading_operand(&chain, above); + perl_count_chain_operands(&chain, absorbed, &mut stats.conditions); } } // An `elsif` is Java's `else if`: the `else` (+1, Rule 5) and @@ -450,7 +546,9 @@ impl Abc for PerlCode { // and adds the three operand slots (issue #1102). P::TernaryExpression => { stats.conditions += 1.; - perl_walk_ternary(node, &mut stats.conditions); + let above = ancestors.iter(node).map(|(ancestor, _)| ancestor); + let absorbed = perl_misparse_absorbs_leading_operand(node, above); + perl_walk_ternary(node, absorbed, &mut stats.conditions); } // Phase-2B (issue #403): condition slots, each read by the // grammar's `condition` field, not at child(1): a comment From f1fcb3ffb2841fc266da6e09ddc06578da2ea2fc Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:39:43 -0700 Subject: [PATCH 06/48] fix(perl): stop scoring an and key as an operator tree-sitter-perl 1.1.2 lexes an auto-quoted bareword `and` as the low-precedence operator. `$h{and}` recovers into an ERROR holding the token, and `(and => 1)` becomes a unary_expression led by it, so cyclomatic, cognitive, ABC and Halstead each scored an operator that is not there. A real `and` always has a left operand. The new lang_helpers::perl::perl_and_is_operator checks that the token's parent is an expression and that the token is not its first child. Each site that keys on `and` now asks it. A rejected `and` is billed as an operand, which is what `$h{or}` already bills. The workaround is marked FIXME(#1539 upstream). Fixes #1539 --- .bca-baseline.toml | 2 +- big-code-analysis-ast/src/getter/perl.rs | 13 ++ big-code-analysis-ast/src/lang_helpers.rs | 1 + .../src/lang_helpers/perl.rs | 49 +++++ src/metrics/abc/perl.rs | 70 ++++--- src/metrics/cognitive/perl.rs | 7 +- src/metrics/cyclomatic/perl.rs | 15 +- src/metrics/mod.rs | 3 + src/metrics/perl_and_key_tests.rs | 173 ++++++++++++++++++ 9 files changed, 291 insertions(+), 42 deletions(-) create mode 100644 big-code-analysis-ast/src/lang_helpers/perl.rs create mode 100644 src/metrics/perl_and_key_tests.rs diff --git a/.bca-baseline.toml b/.bca-baseline.toml index a3d54d44..eef6c303 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -243,7 +243,7 @@ value = 51602.8394982239 path = "big-code-analysis-ast/src/getter/perl.rs" qualified = "PerlCode::get_op_type" metric = "halstead.effort" -value = 106425.53731125958 +value = 113405.25979564841 [[entry]] path = "big-code-analysis-ast/src/getter/php.rs" diff --git a/big-code-analysis-ast/src/getter/perl.rs b/big-code-analysis-ast/src/getter/perl.rs index e2d706ea..8b96adb6 100644 --- a/big-code-analysis-ast/src/getter/perl.rs +++ b/big-code-analysis-ast/src/getter/perl.rs @@ -2,6 +2,7 @@ #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] use super::*; +use crate::lang_helpers::perl::perl_and_is_operator; impl Getter for PerlCode { fn get_space_kind(node: &Node) -> SpaceKind { @@ -37,6 +38,18 @@ impl Getter for PerlCode { { TokenRole::Unknown } + // FIXME(#1539 upstream): an auto-quoted `and` key (`$h{and}`, + // `(and => 1)`) lexes as the operator keyword. It is a + // string key, so it bills what `$h{or}` bills: the + // `identifier` operand the grammar gives every other + // bareword key. + P::And + if !ancestors + .parent(node) + .is_some_and(|parent| perl_and_is_operator(node, &parent)) => + { + TokenRole::Operand + } // Control-flow and declaration keywords. `Perl::Sub` is the // `sub` keyword (token id 16); `Perl::SUB` is the `__SUB__` // literal (token id 7) — that one is an operand, not an diff --git a/big-code-analysis-ast/src/lang_helpers.rs b/big-code-analysis-ast/src/lang_helpers.rs index f0b26341..d23d4685 100644 --- a/big-code-analysis-ast/src/lang_helpers.rs +++ b/big-code-analysis-ast/src/lang_helpers.rs @@ -17,6 +17,7 @@ pub mod elixir; // Crate-private: this dialect's only helper is a kind table the three // classifiers in this crate share, and nothing outside names it. pub(crate) mod irules; +pub mod perl; pub mod python; pub mod tcl; // Crate-private for the same reason as `irules` above: the braced-word diff --git a/big-code-analysis-ast/src/lang_helpers/perl.rs b/big-code-analysis-ast/src/lang_helpers/perl.rs new file mode 100644 index 00000000..a3d1a966 --- /dev/null +++ b/big-code-analysis-ast/src/lang_helpers/perl.rs @@ -0,0 +1,49 @@ +//! Perl: telling the low-precedence `and` operator from an auto-quoted +//! `and` key. + +use crate::Perl; +use crate::node::Node; + +/// Whether `and`, a Perl `and` token whose parent is `parent`, is the +/// low-precedence logical operator. +/// +/// FIXME(#1539 upstream): tree-sitter-perl 1.1.2 lexes an auto-quoted +/// bareword `and` as the keyword. Perl quotes a bareword before `=>` +/// and alone inside a hash subscript, so `(and => 1)` and `$h{and}` +/// hold a string key and no operator, yet the grammar emits two +/// shapes that read as one: +/// +/// - a subscript (`$h{and}`, `$r->{and}`, `$h{-and}`) recovers into an +/// `ERROR` node holding the token; +/// - a fat-comma key (`(and => 1)`, `{ and => 1 }`, `f(x => 1, and => +/// 2)`) becomes a `unary_expression` whose *first* child is the +/// token — the kind a real `$a and $b` also gets, with `$a` first. +/// +/// `and` is an infix operator, so in valid Perl the operator always has +/// a left operand. That is the whole test: the parent is one of the two +/// expression kinds and the token is not its first child, an `extra` +/// aside. +/// `or` and `xor` are unaffected — the grammar reads either as an +/// identifier in both positions. Remove this, and every call site, +/// once the pin carries the upstream fix. +#[must_use] +pub fn perl_and_is_operator(and: &Node, parent: &Node) -> bool { + matches!( + parent.kind_id().into(), + Perl::UnaryExpression | Perl::BinaryExpression + ) && first_non_extra(parent).is_some_and(|first| first.id() != and.id()) +} + +/// Whether `expr` is the `unary_expression` tree-sitter-perl builds +/// around an auto-quoted `and` key (`(and => 1)`): its first child, +/// comments aside, is the `and` token itself. See +/// [`perl_and_is_operator`]. +#[must_use] +pub fn perl_is_and_key_misparse(expr: &Node) -> bool { + first_non_extra(expr).is_some_and(|first| first.kind_id() == Perl::And as u16) +} + +fn first_non_extra<'a>(node: &Node<'a>) -> Option> { + node.children() + .find(|child| !child.as_tree_sitter().is_extra()) +} diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index 69dba7c6..44da5d26 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -13,6 +13,7 @@ use super::{ Abc, Stats, count_boolean_slot, count_each_operand, count_negated_operand, is_operand, last_operand, wrapped_operand, }; +use crate::lang_helpers::perl::perl_and_is_operator; use crate::*; // Fitzpatrick's ABC rules adapted for Perl. @@ -102,33 +103,30 @@ fn perl_condition_scores_itself(expr: &Node) -> bool { match expr.kind_id().into() { P::TernaryExpression | P::PatternMatcher | P::PatternMatcherM => true, - P::BinaryExpression | P::UnaryExpression => expr.children().any(|token| { - matches!( - token.kind_id().into(), - P::EQEQ - | P::BANGEQ - | P::LT - | P::GT - | P::LTEQ - | P::GTEQ - | P::LTEQGT - | P::Eq - | P::Ne - | P::Lt - | P::Gt - | P::Le - | P::Ge - | P::Cmp - | P::EQTILDE - | P::BANGTILDE - | P::AMPAMP - | P::PIPEPIPE - | P::SLASHSLASH - | P::And - | P::Or - | P::Xor - ) - }), + P::BinaryExpression | P::UnaryExpression => { + perl_is_logical_chain(expr) + || expr.children().any(|token| { + matches!( + token.kind_id().into(), + P::EQEQ + | P::BANGEQ + | P::LT + | P::GT + | P::LTEQ + | P::GTEQ + | P::LTEQGT + | P::Eq + | P::Ne + | P::Lt + | P::Gt + | P::Le + | P::Ge + | P::Cmp + | P::EQTILDE + | P::BANGTILDE + ) + }) + } _ => false, } } @@ -242,11 +240,10 @@ fn perl_is_logical_chain(node: &Node) -> bool { matches!( node.kind_id().into(), P::BinaryExpression | P::UnaryExpression - ) && node.children().any(|token| { - matches!( - token.kind_id().into(), - P::AMPAMP | P::PIPEPIPE | P::SLASHSLASH | P::And | P::Or | P::Xor - ) + ) && node.children().any(|token| match token.kind_id().into() { + P::And => perl_and_is_operator(&token, node), + P::AMPAMP | P::PIPEPIPE | P::SLASHSLASH | P::Or | P::Xor => true, + _ => false, }) } @@ -280,12 +277,11 @@ fn perl_walk_ternary(node: &Node, condition_absorbed: bool, conditions: &mut f64 // the two-operand `unary_expression` it parses `$a and $b` as, which the // field-less walk is what scores at all — it scored 0. // -// `leading_absorbed`: see `perl_misparse_absorbs_leading_operand`. +// `leading_absorbed`: see `perl_misparse_absorbs_leading_operand`. The +// `chain` is the operator token's parent, which is no chain when the +// token sits in an `ERROR` or is an auto-quoted `and` key (#1539). fn perl_count_chain_operands(chain: &Node, leading_absorbed: bool, conditions: &mut f64) { - if matches!( - chain.kind_id().into(), - Perl::BinaryExpression | Perl::UnaryExpression - ) { + if perl_is_logical_chain(chain) { for operand in chain .children() .filter(is_operand) diff --git a/src/metrics/cognitive/perl.rs b/src/metrics/cognitive/perl.rs index dff3882c..2243905a 100644 --- a/src/metrics/cognitive/perl.rs +++ b/src/metrics/cognitive/perl.rs @@ -12,6 +12,7 @@ )] use super::*; +use crate::lang_helpers::perl::perl_is_and_key_misparse; /// Folds a Perl `binary_expression`'s (or, for low-precedence `and`, /// `unary_expression`'s) short-circuit operator children @@ -92,8 +93,10 @@ impl Cognitive for PerlCode { // `unary_expression` (with `not`), while `or` / `xor` are a // `binary_expression`; without the second kind every `and` // scored nothing here while cyclomatic and ABC counted it - // (#1530). - P::BinaryExpression | P::UnaryExpression => { + // (#1530). FIXME(#1539 upstream): an auto-quoted `and` key + // (`(and => 1)`) is a `unary_expression` led by that token, + // and holds no operator. + P::BinaryExpression | P::UnaryExpression if !perl_is_and_key_misparse(node) => { compute_perl_booleans(node, stats); } P::FunctionDefinition | P::FunctionDefinitionWithoutSub => { diff --git a/src/metrics/cyclomatic/perl.rs b/src/metrics/cyclomatic/perl.rs index d6fd8c6c..68c9e0a4 100644 --- a/src/metrics/cyclomatic/perl.rs +++ b/src/metrics/cyclomatic/perl.rs @@ -7,12 +7,13 @@ )] use super::*; +use crate::lang_helpers::perl::perl_and_is_operator; impl Cyclomatic for PerlCode { fn compute<'a>( node: &Node<'a>, _code: &'a [u8], - _ancestors: Ancestors<'a, '_>, + ancestors: Ancestors<'a, '_>, stats: &mut Stats, ) { use Perl as P; @@ -41,12 +42,22 @@ impl Cyclomatic for PerlCode { | P::AMPAMPEQ | P::PIPEPIPEEQ | P::SLASHSLASHEQ - | P::And | P::Or | P::TernaryExpression => { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } + // FIXME(#1539 upstream): an auto-quoted `and` key lexes as + // the operator token; only one with a left operand is a + // decision. + P::And + if ancestors + .parent(node) + .is_some_and(|parent| perl_and_is_operator(node, &parent)) => + { + stats.cyclomatic += 1.; + stats.cyclomatic_modified += 1.; + } _ => {} } } diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index b377aa90..7dc126eb 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -69,6 +69,9 @@ mod container_scope_tests; #[cfg(any(feature = "cpp", feature = "mozcpp"))] #[path = "cpp_alternative_tokens_tests.rs"] mod cpp_alternative_tokens_tests; +#[cfg(test)] +#[path = "perl_and_key_tests.rs"] +mod perl_and_key_tests; /// Divides a metric sum by a count, guarding the divisor with `.max(1)`. /// diff --git a/src/metrics/perl_and_key_tests.rs b/src/metrics/perl_and_key_tests.rs new file mode 100644 index 00000000..f4d7b553 --- /dev/null +++ b/src/metrics/perl_and_key_tests.rs @@ -0,0 +1,173 @@ +//! An auto-quoted `and` key scores like any other key, and the +//! low-precedence `and` operator still scores as one (#1539). +//! +//! Perl quotes a bareword before `=>` and alone in a hash subscript, so +//! `$h{and}` and `(and => 1)` hold the string `"and"` and no operator. +//! tree-sitter-perl 1.1.2 lexes that bareword as the keyword anyway, +//! and every metric that keys on the token scored a decision that is +//! not there. `lang_helpers::perl::perl_and_is_operator` tells the two +//! apart; these tests pin the metrics, not the error tree the grammar +//! builds around the key (grammar-dispatch §6). +//! +//! Every row pairs its fixture with a twin that spells the same program +//! without the `and` token — an `or` key, which the grammar reads as a +//! plain identifier, or a `&&` operator — and the twin is the oracle. +//! The headline numbers are asserted outright as well, so a row cannot +//! pass by both spellings drifting together. + +use std::cell::Cell; + +use crate::test_support::check_func_space_only; +use crate::*; + +/// `[cyclomatic, abc conditions, cognitive]` sums of the root space. +/// The cyclomatic sum carries the unit's and the sub's base of 1 each. +#[cfg(feature = "perl")] +type Decisions = [u64; 3]; + +/// `[n1, N1, n2, N2]`. +#[cfg(feature = "perl")] +type Halstead = [u64; 4]; + +#[cfg(feature = "perl")] +fn measure(body: &str) -> (Decisions, Halstead) { + let out = Cell::new(([0; 3], [0; 4])); + check_func_space_only::( + &format!("sub f {{ {body} }}\n"), + "foo.pl", + &[ + Metric::Cyclomatic, + Metric::Abc, + Metric::Cognitive, + Metric::Halstead, + ], + |space| { + let m = &space.metrics; + out.set(( + [ + m.cyclomatic.cyclomatic_sum(), + m.abc.conditions_sum(), + m.cognitive.cognitive_sum(), + ], + [ + m.halstead.unique_operators(), + m.halstead.total_operators(), + m.halstead.unique_operands(), + m.halstead.total_operands(), + ], + )); + }, + ); + out.get() +} + +/// Every valid spelling of an `and` key scores exactly as an `or` key +/// does, in all four metrics. Before #1539 each subscript scored one +/// extra cyclomatic decision and billed `and` as an operator, and each +/// `=>` key also scored an ABC condition and a cognitive sequence. +#[cfg(feature = "perl")] +#[test] +fn perl_and_key_scores_like_an_or_key() { + // `(or_twin, and_key, decisions)`. + let rows: [(&str, &str, Decisions); 7] = [ + ("my %h; return $h{or};", "my %h; return $h{and};", [2, 0, 0]), + ( + "my %h; return $h{ or };", + "my %h; return $h{ and };", + [2, 0, 0], + ), + ( + "my $r; return $r->{or};", + "my $r; return $r->{and};", + [2, 0, 0], + ), + ( + "my %h; if ($h{or}) { return 1; } return 0;", + "my %h; if ($h{and}) { return 1; } return 0;", + [3, 1, 1], + ), + ( + "my %h = (or => 1); return 1;", + "my %h = (and => 1); return 1;", + [2, 0, 0], + ), + ( + "my $h = { or => 1 }; return $h;", + "my $h = { and => 1 }; return $h;", + [2, 0, 0], + ), + ("f(x => 1, or => 2);", "f(x => 1, and => 2);", [2, 0, 0]), + ]; + for (twin, key, decisions) in rows { + assert_eq!( + key.replace("and", "or"), + twin, + "`{key}` drifted from its twin" + ); + let want = measure(twin); + assert_eq!(want.0, decisions, "`{twin}` moved; re-derive the row"); + assert_eq!(measure(key), want, "`{key}` must score like `{twin}`"); + } +} + +/// A list holding an `and` key, assigned in a condition, is one +/// truthiness test: the slot evaluates to the list's last element, so +/// it pays one condition like any other value. +/// +/// This row guards cyclomatic, cognitive and Halstead, but not ABC. The +/// ABC count comes out at 1 whether or not the `and` gate in +/// `perl_is_logical_chain` holds. Without the gate, the slot takes the +/// key for a chain and scores 0, and the chain walk then pays the 1 the +/// slot dropped. `perl_and_key_scores_like_an_or_key` guards that gate. +#[cfg(feature = "perl")] +#[test] +fn perl_and_key_in_a_condition_slot_pays_the_slot() { + let twin = "my %h; if (%h = (or => 1)) { return 1; } return 0;"; + let key = "my %h; if (%h = (and => 1)) { return 1; } return 0;"; + let want = measure(twin); + assert_eq!(want.0, [3, 1, 1], "`{twin}` moved; re-derive the row"); + assert_eq!(measure(key), want, "`{key}` must score like `{twin}`"); +} + +/// A real low-precedence `and` keeps scoring as `&&` does — alone, +/// chained, after `not`, and inside the comparison and ternary shapes +/// the #1273 workaround climbs. Halstead is left out: `and` and `&&` +/// are distinct operators, and the ternary twin needs parentheses. +#[cfg(feature = "perl")] +#[test] +fn perl_and_operator_still_scores_like_ampamp() { + // `(ampamp_twin, and_operator, decisions)`. + let rows: [(&str, &str, Decisions); 5] = [ + ("return ($a && $b);", "return ($a and $b);", [3, 2, 1]), + ( + "return ($a && $b && $c);", + "return ($a and $b and $c);", + [4, 3, 1], + ), + ( + "if (!$a && $b) { return 1; } return 0;", + "if (not $a and $b) { return 1; } return 0;", + [4, 2, 2], + ), + ( + "return ($a > $b && $c);", + "return ($a > $b and $c);", + [3, 2, 1], + ), + ( + "return (($a > $b) && ($c ? 1 : 2));", + "return ($a > $b and $c ? 1 : 2);", + [4, 3, 2], + ), + ]; + for (twin, operator, decisions) in rows { + assert!(operator.contains(" and "), "`{operator}` lost its `and`"); + let want = measure(twin).0; + assert_eq!(want, decisions, "`{twin}` moved; re-derive the row"); + assert_eq!( + measure(operator).0, + want, + "`{operator}` must score like `{twin}`" + ); + } +} From e7271435afd52b79ba2cf41ed40bef37bfe3f750 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:01:42 -0700 Subject: [PATCH 07/48] fix(abc/perl): stop scoring =~ s/// as a test by use `=~` / `!~` counted as a condition whatever they bound, so `$x =~ s/a/b/;` scored 1 while the bare `s/a/b/;` on `$_` scored 0. A substitution or transliteration edits its string and yields a count, so by use it is no test in either spelling, as #1467 already decided for the bare form. `perl_binding_rewrites` reads the binding's right operand and gates both the operator-token arm and `perl_condition_scores_itself` on it, so a slot still pays for a rewrite binding exactly once and every `if (...)` reading is unchanged. Matches (`/re/`, `m{}`, `qr//`, a pattern variable) keep their 1. `!~ s///r` / `!~ tr///r` are Perl compile errors and are documented, not pinned. Fixes #1540 --- big-code-analysis-book/src/metrics.md | 2 +- src/metrics/abc.rs | 79 ++++++++++++++++++++++++++- src/metrics/abc/perl.rs | 56 ++++++++++++++----- src/metrics/abc/slot_pays_tests.rs | 16 +++++- 4 files changed, 134 insertions(+), 19 deletions(-) diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index e5a6b7d2..88969aeb 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -159,7 +159,7 @@ application would over-count. | Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive` (its `after` timeout included), of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct, and a `with` with no `else` now scores nothing, level with cyclomatic. The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). | | C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these `<` / `>` as an operator, and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | -| C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | +| C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | | Perl | A comparison's right operand is not a condition, although the grammar parses it as one | tree-sitter-perl 1.1.2 lets a chaining comparison (`<`, `>`, `<=`, `>=`, `lt`, `gt`, `le`, `ge`, `==`, `!=`, `eq`, `ne`) take everything of lower precedence to its right as its operand, so `$a > $b ? 1 : 2` parses as `$a > ($b ? 1 : 2)` and `$a > $b && $c` as `$a > ($b && $c)`. The ternary's condition slot and the chain's first operand then held `$b`, which scored as a condition of its own: one above the JavaScript twin for each misparsed comparison. The walker recognises the shape and leaves that operand to the comparison, so both spellings score what the parenthesised `($a > $b) ? 1 : 2` and `($a > $b) && $c` do. An explicit `$a > ($b ? 1 : 2)` is unaffected. `<=>`, `cmp`, `=~` and `!~` do not chain and parse correctly. The workaround goes when the grammar is fixed upstream (#1273). | #### Where a constructor call lands {#abc-constructor-attribution} diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index 3499c978..746ce49f 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -14536,8 +14536,8 @@ end // reads what it read before. `bound_out` / `nbound` pin the gate on // `=~` / `!~` — the operator token scores, the pattern under it does // not — and `ctrl_out` is the `==` control the rest level with. - // `s///`, `tr///` and `qr//` are not matches and stay at 0; whether - // the first two should score by use is #1540. + // `s///`, `tr///` and `qr//` are not matches and stay at 0; the + // bound `s///` / `tr///` agree (#1540, the test below). #[cfg(feature = "perl")] #[test] fn perl_bare_match_is_a_condition_by_use() { @@ -14597,6 +14597,81 @@ end ); } + // A substitution or transliteration bound with `=~` / `!~` is the + // same edit as its bare form on `$_`, so it scores by use exactly as + // that twin does: 0 (#1540). Each `*_s` / `*_tr` / `*_y` row below + // was 1 through the operator token before the gate; `count` (an + // assigned count) and `sr` (a `/r` copy, a value) are uses too. A + // match keeps its 1 whatever spells it — a pattern, `qr//`, or a + // variable holding one. The `if_*` rows are the slot side: each pays + // once, as its bare twin `if_s` does, because + // `perl_condition_scores_itself` stops claiming a rewrite binding in + // the same change; `tern_s` and `neg_s` reach that predicate through + // a ternary condition and a `!`-proved `return` operand. + #[cfg(feature = "perl")] + #[test] + fn perl_bound_rewrite_scores_like_its_bare_twin() { + check_func_space::( + "sub ctrl { my ($x) = @_; $x; return $x; }\n\ + sub m_bare { my ($x) = @_; /re/; return $x; }\n\ + sub m_bound { my ($x) = @_; $x =~ /re/; return $x; }\n\ + sub m_nbound { my ($x) = @_; $x !~ /re/; return $x; }\n\ + sub m_qr { my ($x) = @_; $x =~ qr/re/; return $x; }\n\ + sub m_var { my ($x, $q) = @_; $x =~ $q; return $x; }\n\ + sub s_bare { my ($x) = @_; s/a/b/; return $x; }\n\ + sub s_bound { my ($x) = @_; $x =~ s/a/b/; return $x; }\n\ + sub s_nbound { my ($x) = @_; $x !~ s/a/b/; return $x; }\n\ + sub tr_bare { my ($x) = @_; tr/a/b/; return $x; }\n\ + sub tr_bound { my ($x) = @_; $x =~ tr/a/b/; return $x; }\n\ + sub tr_nbound { my ($x) = @_; $x !~ tr/a/b/; return $x; }\n\ + sub y_bound { my ($x) = @_; $x =~ y/a/b/; return $x; }\n\ + sub count { my ($x) = @_; my $n = ($x =~ s/a/b/g); return $x; }\n\ + sub sr { my ($x) = @_; my $r = $x =~ s/a/b/r; return $r; }\n\ + sub if_s { my ($x) = @_; if (s/a/b/) { g(); } return $x; }\n\ + sub if_m { my ($x) = @_; if ($x =~ /re/) { g(); } return $x; }\n\ + sub if_bs { my ($x) = @_; if ($x =~ s/a/b/) { g(); } return $x; }\n\ + sub if_nbs { my ($x) = @_; if ($x !~ s/a/b/) { g(); } return $x; }\n\ + sub if_btr { my ($x) = @_; if ($x =~ tr/a/b/) { g(); } return $x; }\n\ + sub if_by { my ($x) = @_; if ($x =~ y/a/b/) { g(); } return $x; }\n\ + sub if_bsr { my ($x) = @_; if ($x =~ s/a/b/r) { g(); } return $x; }\n\ + sub tern_s { my ($x) = @_; return ($x =~ s/a/b/) ? 1 : 0; }\n\ + sub neg_s { my ($x) = @_; return !($x =~ s/a/b/); }\n", + "foo.pl", + |space| { + assert_members_score( + &space, + &[ + ("ctrl", 0, 1), + ("m_bare", 1, 1), + ("m_bound", 1, 1), + ("m_nbound", 1, 1), + ("m_qr", 1, 1), + ("m_var", 1, 1), + ("s_bare", 0, 1), + ("s_bound", 0, 1), + ("s_nbound", 0, 1), + ("tr_bare", 0, 1), + ("tr_bound", 0, 1), + ("tr_nbound", 0, 1), + ("y_bound", 0, 1), + ("count", 0, 1), + ("sr", 0, 1), + ("if_s", 1, 2), + ("if_m", 1, 2), + ("if_bs", 1, 2), + ("if_nbs", 1, 2), + ("if_btr", 1, 2), + ("if_by", 1, 2), + ("if_bsr", 1, 2), + // The ternary node plus its condition slot. + ("tern_s", 2, 2), + ("neg_s", 1, 1), + ], + ); + }, + ); + } + // The pattern `split` takes first is its delimiter, not a match // against `$_`, so it is no condition (#1467) in any spelling: the // spaced and parenthesised call forms, the `m{}` pattern, and the diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index 44da5d26..6793c2d8 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -33,7 +33,8 @@ use crate::*; // already contributed the branch. // - Conditions: numeric and string comparison operators (`==`, `!=`, // `<`, `>`, `<=`, `>=`, `<=>`, `eq`, `ne`, `lt`, `gt`, `le`, `ge`, -// `cmp`, `=~`, `!~`), a bare `/re/` / `m{re}` match against `$_`, +// `cmp`, and `=~` / `!~` binding a match rather than an `s///` / +// `tr///` edit), a bare `/re/` / `m{re}` match against `$_`, // the ternary operator (`TernaryExpression`), // and each `elsif` / `else` clause of an `if` / `unless` // statement. Each condition slot pays one condition unless an arm @@ -105,9 +106,10 @@ fn perl_condition_scores_itself(expr: &Node) -> bool { P::TernaryExpression | P::PatternMatcher | P::PatternMatcherM => true, P::BinaryExpression | P::UnaryExpression => { perl_is_logical_chain(expr) - || expr.children().any(|token| { - matches!( - token.kind_id().into(), + || expr.children().any(|token| match token.kind_id().into() { + P::EQTILDE | P::BANGTILDE => !perl_binding_rewrites(expr), + kind => matches!( + kind, P::EQEQ | P::BANGEQ | P::LT @@ -122,15 +124,35 @@ fn perl_condition_scores_itself(expr: &Node) -> bool { | P::Le | P::Ge | P::Cmp - | P::EQTILDE - | P::BANGTILDE - ) + ), }) } _ => false, } } +// Whether a `=~` / `!~` binding applies a substitution or transliteration +// (`$x =~ s/a/b/`, `$x =~ tr/a/b/`, `$x =~ y/a/b/`) rather than a match. +// Those edit `$x` and yield a count, so by use they are no test, exactly +// as the bare `s///` / `tr///` on `$_` are not (#1540) — while a match +// (`/re/`, `m{re}`, `qr//`, or a variable holding a pattern) is. A slot +// still pays for a rewrite binding through `perl_count_condition`, +// because this predicate also keeps `perl_condition_scores_itself` from +// claiming it. The grammar names no `left` / `right` field on a +// `binary_expression`, so the right operand is read as its last. +// +// `$x !~ s///r` and `$x !~ tr///r` parse the same way but are Perl +// compile errors ("Using !~ with s///r doesn't make sense"); valid input +// cannot tell how they score, so no test pins them (grammar-dispatch §6). +fn perl_binding_rewrites(binding: &Node) -> bool { + last_operand(binding).is_some_and(|rhs| { + matches!( + rhs.kind_id().into(), + Perl::SubstitutionPatternS | Perl::TransliterationTrOrY + ) + }) +} + // Scores one boolean slot — an `if` / `elsif` / `unless` / `while` / // `until` condition, a statement modifier's condition, a ternary or // C-style `for` condition, an operand of a logical chain (see @@ -378,7 +400,8 @@ impl Abc for PerlCode { // grammar can hand us, neither being reasoning a reader must // do. Adding the guarded `<` / `>` arm for #1297 took the // count from 14 to 15, the statement-modifier arm for #1464 - // to 16, and the bare-match arm for #1467 to 17; each arm is + // to 16, the bare-match arm for #1467 to 17, and the `=~` / + // `!~` rewrite gate for #1540 added one more; each arm is // independent and self-describing like every other, and there // is no semantic boundary to split this lookup on. use Perl as P; @@ -453,11 +476,18 @@ impl Abc for PerlCode { | P::Le | P::Ge | P::Cmp - | P::EQTILDE - | P::BANGTILDE | P::ElseClause => { stats.conditions += 1.; } + // `=~` / `!~` bind a match, a test, or a rewrite, an edit + // that scores only in a slot (see `perl_binding_rewrites`). + P::EQTILDE | P::BANGTILDE + if !ancestors + .parent(node) + .is_some_and(|binding| perl_binding_rewrites(&binding)) => + { + stats.conditions += 1.; + } // Counts `<` / `>` only as the operator token of a // `binary_expression`, the allowlist polarity the rest of // the workspace uses. Ungated, a readline scored two @@ -487,9 +517,9 @@ impl Abc for PerlCode { // `my $r = ($x =~ /^#/)`. `perl_condition_scores_itself` // lists it, or every slot would score it a second time. // `s///` and `tr///` stay out: they edit `$_` and yield a - // count. A slot still pays for either (#1475), but by use the - // bound `$x =~ s///` scores through `=~` and the bare form - // does not — the disagreement #1540 tracks. + // count, so by use they are no test. A slot still pays for + // either (#1475), and the bound `$x =~ s///` agrees with the + // bare form (#1540, `perl_binding_rewrites`). P::PatternMatcher | P::PatternMatcherM if !perl_pattern_is_bound_or_delimiter(node, code, ancestors) => { diff --git a/src/metrics/abc/slot_pays_tests.rs b/src/metrics/abc/slot_pays_tests.rs index 6b6b3644..6e39403c 100644 --- a/src/metrics/abc/slot_pays_tests.rs +++ b/src/metrics/abc/slot_pays_tests.rs @@ -1055,8 +1055,8 @@ fn perl_elsif_is_an_else_and_a_slot() { /// (all three spellings), an anonymous sub, and the array and hash /// variables in both their named and dereferencing forms. `@$b` was /// the case #1475 called the clearest defect, since `@a` scored and it -/// did not. Each pays a slot; whether `s///` and `tr///` count outside -/// one is a separate question (#1540), which this test does not settle. +/// did not. Each pays a slot, the `=~` / `!~`-bound rewrites included: +/// outside one they score nothing (#1540), so the slot is all they pay. #[test] #[cfg(feature = "perl")] fn perl_1475_constructs_pay_their_slot() { @@ -1069,7 +1069,17 @@ fn perl_1475_constructs_pay_their_slot() { ], "$b", &[ - "s/x/y/", "tr/a/b/", "y/a/b/", "sub {1}", "@a", "@$b", "%h", "%$h", + "s/x/y/", + "tr/a/b/", + "y/a/b/", + "$b =~ s/x/y/", + "$b !~ s/x/y/", + "$b =~ tr/a/b/", + "sub {1}", + "@a", + "@$b", + "%h", + "%$h", ], ); } From 1ba327ce07b57527b114451d2119857eb7a4e2e3 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:17:15 -0700 Subject: [PATCH 08/48] fix(perl): stop billing a not key as an operator tree-sitter-perl 1.1.2 lexes an auto-quoted bareword `not` as the keyword. `$h{not}` recovers into an ERROR holding the token, and `(not => 1)` into the unary_expression a real `not 1` gets, with the `=>` wrapped in an ERROR beside it. The getter billed both as the `not` operator where the `or` twin bills an identifier operand, and ABC's value peel read a `=>` key as a negation, so `f(not => 1)` and `return (not => 1)` scored a condition. `not` is a prefix operator, so #1539's left-operand test cannot tell the two apart; perl_not_is_key reads the recovery node instead. The skipped `=>` sits in an ERROR the parser marks as an extra, so the sibling scan keeps it while still passing over comments. The #1539 test module now covers both keywords and is renamed to perl_keyword_key_tests.rs. Fixes #1541 --- .bca-baseline.toml | 2 +- big-code-analysis-ast/src/getter/perl.rs | 11 +- .../src/lang_helpers/perl.rs | 39 ++++- src/metrics/abc/perl.rs | 20 ++- src/metrics/mod.rs | 4 +- ...key_tests.rs => perl_keyword_key_tests.rs} | 153 ++++++++++++++++-- 6 files changed, 204 insertions(+), 25 deletions(-) rename src/metrics/{perl_and_key_tests.rs => perl_keyword_key_tests.rs} (52%) diff --git a/.bca-baseline.toml b/.bca-baseline.toml index eef6c303..a3b52354 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -243,7 +243,7 @@ value = 51602.8394982239 path = "big-code-analysis-ast/src/getter/perl.rs" qualified = "PerlCode::get_op_type" metric = "halstead.effort" -value = 113405.25979564841 +value = 120979.88227622502 [[entry]] path = "big-code-analysis-ast/src/getter/php.rs" diff --git a/big-code-analysis-ast/src/getter/perl.rs b/big-code-analysis-ast/src/getter/perl.rs index 8b96adb6..9691d5ff 100644 --- a/big-code-analysis-ast/src/getter/perl.rs +++ b/big-code-analysis-ast/src/getter/perl.rs @@ -2,7 +2,7 @@ #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] use super::*; -use crate::lang_helpers::perl::perl_and_is_operator; +use crate::lang_helpers::perl::{perl_and_is_operator, perl_not_is_key}; impl Getter for PerlCode { fn get_space_kind(node: &Node) -> SpaceKind { @@ -50,6 +50,15 @@ impl Getter for PerlCode { { TokenRole::Operand } + // FIXME(#1541 upstream): the same for an auto-quoted `not` + // key (`$h{not}`, `(not => 1)`); see `perl_not_is_key`. + P::Not + if ancestors + .parent(node) + .is_some_and(|parent| perl_not_is_key(node, &parent)) => + { + TokenRole::Operand + } // Control-flow and declaration keywords. `Perl::Sub` is the // `sub` keyword (token id 16); `Perl::SUB` is the `__SUB__` // literal (token id 7) — that one is an operand, not an diff --git a/big-code-analysis-ast/src/lang_helpers/perl.rs b/big-code-analysis-ast/src/lang_helpers/perl.rs index a3d1a966..029c88ed 100644 --- a/big-code-analysis-ast/src/lang_helpers/perl.rs +++ b/big-code-analysis-ast/src/lang_helpers/perl.rs @@ -1,5 +1,5 @@ -//! Perl: telling the low-precedence `and` operator from an auto-quoted -//! `and` key. +//! Perl: telling the `and` and `not` operators from an auto-quoted +//! `and` or `not` key. use crate::Perl; use crate::node::Node; @@ -43,6 +43,41 @@ pub fn perl_is_and_key_misparse(expr: &Node) -> bool { first_non_extra(expr).is_some_and(|first| first.kind_id() == Perl::And as u16) } +/// Whether `not`, a Perl `not` token whose parent is `parent`, is an +/// auto-quoted hash key rather than the prefix operator. +/// +/// FIXME(#1541 upstream): the `not` sibling of #1539 — see +/// [`perl_and_is_operator`]. tree-sitter-perl 1.1.2 lexes a bareword +/// `not` before `=>` or alone in a hash subscript as the keyword, and +/// recovers into one of two shapes: +/// +/// - a subscript (`$h{not}`, `$r->{not}`) gives an `ERROR` node holding +/// the token; +/// - a fat-comma key (`(not => 1)`, `{ not => 1 }`) gives the +/// `unary_expression` a real `not 1` gets, with an `ERROR` wrapping +/// the `=>` as the token's next sibling. +/// +/// `and`'s test does not carry over: `not` is a prefix operator, so it +/// is the first child of its expression whether it is a key or not. The +/// recovery node is the only signal, so this reads it directly. A real +/// `not` that lands inside an `ERROR` for an unrelated reason is billed +/// as a key too; that takes input the grammar already failed to parse. +/// `or` is unaffected — the grammar reads it as an identifier in both +/// positions. Remove this, and every call site, once the pin carries the +/// upstream fix. +#[must_use] +pub fn perl_not_is_key(not: &Node, parent: &Node) -> bool { + parent.is_error() + || parent + .children() + // The parser marks the `ERROR` it wraps a skipped `=>` in + // as an extra, so only comments are passed over here. + .filter(|child| child.is_error() || !child.as_tree_sitter().is_extra()) + .skip_while(|child| child.id() != not.id()) + .nth(1) + .is_some_and(|next| next.is_error() && next.is_child(Perl::FatComma as u16)) +} + fn first_non_extra<'a>(node: &Node<'a>) -> Option> { node.children() .find(|child| !child.as_tree_sitter().is_extra()) diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index 6793c2d8..af90f10a 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -13,7 +13,7 @@ use super::{ Abc, Stats, count_boolean_slot, count_each_operand, count_negated_operand, is_operand, last_operand, wrapped_operand, }; -use crate::lang_helpers::perl::perl_and_is_operator; +use crate::lang_helpers::perl::{perl_and_is_operator, perl_not_is_key}; use crate::*; // Fitzpatrick's ABC rules adapted for Perl. @@ -68,6 +68,11 @@ use crate::*; // match the hidden `_unary_not` supertype (`P::UnaryNot`), which the // parser never emits (grammar-dispatch item 2). // +// FIXME(#1541 upstream): an auto-quoted `not` key (`f(not => !$a)`) +// parses as that same `unary_expression`, with the `=>` in an `ERROR` +// beside the token. It is the list `("not", !$a)`, not a negation, so +// it peels like one: to its last element, the value, proving nothing. +// // No operand is read by index: a comment may sit before or after it // (`(# c` / `! # c`, #1455). fn perl_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { @@ -79,12 +84,13 @@ fn perl_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { P::BinaryExpression if node.is_child(P::EQ as u16) => { last_operand(node).map(|o| (o, false)) } - P::UnaryExpression - if node - .child_by_field_name("operator") - .is_some_and(|op| matches!(op.kind_id().into(), P::BANG | P::Not)) => - { - wrapped_operand(node).map(|o| (o, true)) + P::UnaryExpression => { + node.child_by_field_name("operator") + .and_then(|op| match op.kind_id().into() { + P::Not if perl_not_is_key(&op, node) => last_operand(node).map(|o| (o, false)), + P::BANG | P::Not => wrapped_operand(node).map(|o| (o, true)), + _ => None, + }) } _ => None, } diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 7dc126eb..c397e0b3 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -70,8 +70,8 @@ mod container_scope_tests; #[path = "cpp_alternative_tokens_tests.rs"] mod cpp_alternative_tokens_tests; #[cfg(test)] -#[path = "perl_and_key_tests.rs"] -mod perl_and_key_tests; +#[path = "perl_keyword_key_tests.rs"] +mod perl_keyword_key_tests; /// Divides a metric sum by a count, guarding the divisor with `.max(1)`. /// diff --git a/src/metrics/perl_and_key_tests.rs b/src/metrics/perl_keyword_key_tests.rs similarity index 52% rename from src/metrics/perl_and_key_tests.rs rename to src/metrics/perl_keyword_key_tests.rs index f4d7b553..ca505cc0 100644 --- a/src/metrics/perl_and_key_tests.rs +++ b/src/metrics/perl_keyword_key_tests.rs @@ -1,19 +1,20 @@ -//! An auto-quoted `and` key scores like any other key, and the -//! low-precedence `and` operator still scores as one (#1539). +//! An auto-quoted `and` or `not` key scores like any other key, and the +//! `and` and `not` operators still score as operators (#1539, #1541). //! //! Perl quotes a bareword before `=>` and alone in a hash subscript, so -//! `$h{and}` and `(and => 1)` hold the string `"and"` and no operator. -//! tree-sitter-perl 1.1.2 lexes that bareword as the keyword anyway, -//! and every metric that keys on the token scored a decision that is -//! not there. `lang_helpers::perl::perl_and_is_operator` tells the two -//! apart; these tests pin the metrics, not the error tree the grammar -//! builds around the key (grammar-dispatch §6). +//! `$h{and}` and `(not => 1)` hold a string key and no operator. +//! tree-sitter-perl 1.1.2 lexes either bareword as the keyword anyway, +//! and every metric that keys on the token scored an operator, or a +//! decision, that is not there. `lang_helpers::perl::perl_and_is_operator` +//! and `perl_not_is_key` tell the two apart; these tests pin the metrics, +//! not the error tree the grammar builds around the key +//! (grammar-dispatch §6). //! //! Every row pairs its fixture with a twin that spells the same program -//! without the `and` token — an `or` key, which the grammar reads as a -//! plain identifier, or a `&&` operator — and the twin is the oracle. -//! The headline numbers are asserted outright as well, so a row cannot -//! pass by both spellings drifting together. +//! without the keyword token — an `or` key, which the grammar reads as a +//! plain identifier, or a `&&` / `!` operator — and the twin is the +//! oracle. The headline numbers are asserted outright as well, so a row +//! cannot pass by both spellings drifting together. use std::cell::Cell; @@ -171,3 +172,131 @@ fn perl_and_operator_still_scores_like_ampamp() { ); } } + +/// Every valid spelling of a `not` key scores exactly as an `or` key +/// does. Before #1541 each billed `not` as an operator where the twin +/// bills an operand, and a `=>` key outside a boolean slot also scored +/// an ABC condition, the peel reading it as `!` applied to its value. +/// The peel must still reach that value, which a negation there proves +/// boolean like any other. +#[cfg(feature = "perl")] +#[test] +fn perl_not_key_scores_like_an_or_key() { + // `(or_twin, not_key, decisions, halstead)`. + let rows: [(&str, &str, Decisions, Halstead); 10] = [ + ( + "my %h; return $h{or};", + "my %h; return $h{not};", + [2, 0, 0], + [6, 8, 4, 4], + ), + ( + "my %h; return $h{ or };", + "my %h; return $h{ not };", + [2, 0, 0], + [6, 8, 4, 4], + ), + ( + "my $r; return $r->{or};", + "my $r; return $r->{not};", + [2, 0, 0], + [7, 10, 3, 4], + ), + ( + "my %h = (or => 1); return 1;", + "my %h = (not => 1); return 1;", + [2, 0, 0], + [8, 9, 4, 5], + ), + ( + "my $r = { or => 1 }; return 1;", + "my $r = { not => 1 }; return 1;", + [2, 0, 0], + [8, 10, 4, 5], + ), + ( + "f(x => 1, or => 2);", + "f(x => 1, not => 2);", + [2, 0, 0], + [6, 7, 5, 6], + ), + ("f(or => 1);", "f(not => 1);", [2, 0, 0], [5, 5, 3, 4]), + ( + "return (or => 1);", + "return (not => 1);", + [2, 0, 0], + [6, 6, 3, 3], + ), + // The key's value is still a slot: the `!` proves it boolean. + ("f(or => !$a);", "f(not => !$a);", [2, 1, 0], [7, 7, 3, 4]), + ( + "my $y = $c ? 1 : (or => 2);", + "my $y = $c ? 1 : (not => 2);", + [3, 2, 1], + [10, 11, 6, 6], + ), + ]; + for (twin, key, decisions, halstead) in rows { + assert_eq!( + key.replace("not", "or"), + twin, + "`{key}` drifted from its twin" + ); + let want = measure(twin); + assert_eq!( + want, + (decisions, halstead), + "`{twin}` moved; re-derive the row" + ); + assert_eq!(measure(key), want, "`{key}` must score like `{twin}`"); + } +} + +/// A real `not` keeps scoring as `!` does, in all four metrics: alone, +/// in a condition, applied to a list, as a call argument the negation +/// proves boolean, ahead of a low-precedence `and`, and as the value of +/// a `not` key, where the key's recovery node sits one level up. `not` +/// and `!` are distinct operators, but each fixture spells only one of +/// them, so the Halstead counts agree too. +#[cfg(feature = "perl")] +#[test] +fn perl_not_operator_still_scores_like_bang() { + // `(bang_twin, not_operator, decisions, halstead)`. + let rows: [(&str, &str, Decisions, Halstead); 6] = [ + ("!$a;", "not $a;", [2, 0, 0], [5, 5, 2, 2]), + ( + "if (!$a) { return 1; } return 0;", + "if (not $a) { return 1; } return 0;", + [3, 1, 1], + [8, 11, 4, 4], + ), + ("!($a);", "not($a);", [2, 0, 0], [6, 6, 2, 2]), + ("f(!$a);", "f(not $a);", [2, 1, 0], [6, 6, 2, 3]), + ( + "return (!$a && $b);", + "return (not $a and $b);", + [3, 2, 1], + [8, 9, 3, 3], + ), + ( + "my %h = (or => !$a); return 1;", + "my %h = (not => not $a); return 1;", + [2, 0, 0], + [10, 11, 5, 5], + ), + ]; + for (twin, operator, decisions, halstead) in rows { + assert!(operator.contains("not"), "`{operator}` lost its `not`"); + let want = measure(twin); + assert_eq!( + want, + (decisions, halstead), + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure(operator), + want, + "`{operator}` must score like `{twin}`" + ); + } +} From 50ced6a05e09576b8790e8c172235ba0093d2a9d Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:37:51 -0700 Subject: [PATCH 09/48] fix(metrics): score xor as a value, not a decision Perl and PHP `xor` evaluates both operands, so like Java `^`, C++ `xor` and Kotlin's infix `xor` it is a value. Perl ABC walked it as a chain (`if ($a xor $b)` scored 2 against cyclomatic 2), and PHP counted it in all three metrics (3 / 2 / 2). Both now score as their `^` twin: 2 / 1 / 1 in an `if`, 1 / 0 / 0 by use. Halstead still bills `xor` as an operator. Refs #1536 --- big-code-analysis-book/src/metrics.md | 10 +++--- src/metrics/abc.rs | 25 ++++++++------- src/metrics/abc/perl.rs | 18 +++++++---- src/metrics/abc/php.rs | 15 +++++---- src/metrics/abc/slot_pays_tests.rs | 7 +++- src/metrics/cognitive.rs | 32 +++++++++++-------- src/metrics/cognitive/php.rs | 16 ++++++---- src/metrics/cognitive/word_operator_tests.rs | 14 ++++---- src/metrics/cyclomatic/php.rs | 5 +-- ...t_not_counted_directly_ternary_counts.snap | 8 ++--- 10 files changed, 88 insertions(+), 62 deletions(-) diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 88969aeb..cc8d449a 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -116,10 +116,12 @@ each row attributed to the figure that introduces it. The short-circuit logical operators (`&&`, `||`, and per-language equivalents — Ruby `and` / `or`, Python `and` / `or`, Perl `and` / -`or` / `xor`, Lua `and` / `or`, Tcl `&&` / `||`, iRules `&&` / `||` / +`or`, Lua `and` / `or`, Tcl `&&` / `||`, iRules `&&` / `||` / `and` / `or`) do **not** contribute a condition on their own. Each non-comparison operand -contributes one instead, via the unary-conditional rule. The +contributes one instead, via the unary-conditional rule. Perl and +PHP `xor` evaluate both operands, so like `^` they are a value: the +slot holding one pays a single condition. The paper makes this explicit twice: 1. **Listing 2** annotates `(am >= 0 && am <= 0xF) ? '/' : 'C'` as @@ -335,8 +337,8 @@ where the cognitive load jumps, so the second operator earns its own token: where a language spells conjunction and disjunction both ways (C++, Ruby, PHP, Perl, Elixir, iRules), `a && b and c` is one sequence. Precedence still decides what nests where, so Ruby's -`a || b and c` is `(a || b) and c` and scores two. `xor`, `??` and -Perl's `//` are operations of their own. +`a || b and c` is `(a || b) and c` and scores two. `??` and Perl's +`//` are operations of their own; `xor`, like `^`, is no sequence. big-code-analysis exports the per-function structural score along with the file-wide `sum`, `min`, `max`, and a per-function `average`. diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index 746ce49f..37859660 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -80,7 +80,7 @@ mod tcl; /// Java `default:` precedent). The short- /// circuit logical operators `&&` and `||` (and per-language /// equivalents — Python's `and` / `or`, Lua's `and` / `or`, Tcl's -/// `&&` / `||`, Perl's `&&` / `||` / `//` / `and` / `or` / `xor`) +/// `&&` / `||`, Perl's `&&` / `||` / `//` / `and` / `or`) /// are deliberately **not** counted on their own. The paper's /// worked Listing 2 annotates `(am >= 0 && am <= 0xF) ? '/' : 'C'` /// as `accc` — three conditions for `>=`, `<=`, `?`, zero for @@ -14057,10 +14057,12 @@ end #[cfg(feature = "perl")] #[test] fn perl_short_circuit_not_counted_directly_ternary_counts() { - // `&&`, `||`, `//`, low-precedence `and`, `or`, `xor` are - // NOT counted as conditions on their own (Fitzpatrick Rule - // 5; #395) — instead each operand is counted as a unary - // conditional by the walker (Rule 9; #403). The pinned + // `&&`, `||`, `//`, low-precedence `and`, `or` are NOT + // counted as conditions on their own (Fitzpatrick Rule 5; + // #395) — instead each operand is counted as a unary + // conditional by the walker (Rule 9; #403). `xor` evaluates + // both operands, so it is a value like `^` and, outside a + // slot, scores nothing (#1536). The pinned // tree-sitter-perl parses the low-precedence `and` as a // two-operand `unary_expression` rather than a // `binary_expression`; the walker reads the operands of either @@ -14068,8 +14070,9 @@ end // The low-precedence forms bind looser than `=`, so // `$r = $a and $b` is `($r = $a) and $b`, and its left operand // pays through the assignment's value. - // Net: 6 chain lines × 2 operands + 1 ternary node + 1 for the - // ternary's bare `$a` condition operand (#1102) = 14. + // Net: 5 chain lines × 2 operands + 0 for the `xor` line + 1 + // ternary node + 1 for the ternary's bare `$a` condition operand + // (#1102) = 12. check_metrics::( "sub f {\n\ my $r;\n\ @@ -14086,9 +14089,9 @@ end // 7 `=` tokens (one per reassignment line). assert_eq!(metric.abc.assignments_sum(), 7); assert_eq!(metric.abc.branches_sum(), 0); - // 6 chain lines × 2 operands + 1 ternary node + 1 for - // its bare `$a` condition operand = 14. - assert_eq!(metric.abc.conditions_sum(), 14); + // 5 chain lines × 2 operands + 1 ternary node + 1 for + // its bare `$a` condition operand = 12. + assert_eq!(metric.abc.conditions_sum(), 12); insta::assert_json_snapshot!(metric.abc); }, ); @@ -14311,7 +14314,7 @@ end #[test] fn perl_if_multiple_conditions() { // Fitzpatrick Rule 9 walker (issue #403): each operand of a - // `&&` / `||` / `//` / `and` / `or` / `xor` chain is one + // `&&` / `||` / `//` / `and` / `or` chain is one // condition. ScalarVariable operands ($a, $b, …) qualify as // terminal-bool kinds for the walker. check_metrics::( diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index af90f10a..6f289874 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -41,11 +41,14 @@ use crate::*; // inside it already did (see `perl_count_condition`). // // The short-circuit and low-precedence logical operators (`&&`, -// `||`, `//`, `and`, `or`, `xor`) are deliberately NOT counted. +// `||`, `//`, `and`, `or`) are not counted as tokens; each operand +// of their chain is a boolean slot instead (Rule 9, issue #403). // See the module-level `Stats` doc-comment for the cross- // language policy (Fitzpatrick rules mapped from Figure 2 for C, // the closest analogue since the paper does not define rules for -// Perl; issue #395, walker tracked in #403). +// Perl; issue #395). `xor` is no chain: it evaluates both operands, +// so like Java's `^` it is a value, and the slot holding it pays +// one condition (#1536). // One step of the value peel (see `PeelStep`). // // `Array` is tree-sitter-perl's name for the `(...)` shape used BOTH as @@ -260,8 +263,9 @@ fn perl_misparse_absorbs_leading_operand<'a>( }) } -// A `&&` / `||` / `//` / `and` / `or` / `xor` chain. tree-sitter-perl -// spells `and` as a two-operand `unary_expression`. +// A `&&` / `||` / `//` / `and` / `or` chain. tree-sitter-perl spells +// `and` as a two-operand `unary_expression`. `xor` is left out: it does +// not short-circuit, so it is a value, as `^` is (#1536). fn perl_is_logical_chain(node: &Node) -> bool { use Perl as P; @@ -270,7 +274,7 @@ fn perl_is_logical_chain(node: &Node) -> bool { P::BinaryExpression | P::UnaryExpression ) && node.children().any(|token| match token.kind_id().into() { P::And => perl_and_is_operator(&token, node), - P::AMPAMP | P::PIPEPIPE | P::SLASHSLASH | P::Or | P::Xor => true, + P::AMPAMP | P::PIPEPIPE | P::SLASHSLASH | P::Or => true, _ => false, }) } @@ -534,8 +538,8 @@ impl Abc for PerlCode { // Fitzpatrick Rule 9 walker: each operand of a Perl // short-circuit / low-precedence logical chain is one // condition (issue #403). Covers `&&`, `||`, `//`, - // `and`, `or`, `xor`. - P::AMPAMP | P::PIPEPIPE | P::SLASHSLASH | P::And | P::Or | P::Xor => { + // `and`, `or` — not `xor`, which is a value (#1536). + P::AMPAMP | P::PIPEPIPE | P::SLASHSLASH | P::And | P::Or => { let mut above = ancestors.iter(node).map(|(ancestor, _)| ancestor); if let Some(chain) = above.next() { let absorbed = perl_misparse_absorbs_leading_operand(&chain, above); diff --git a/src/metrics/abc/php.rs b/src/metrics/abc/php.rs index dc1ce2e5..70fcdb8c 100644 --- a/src/metrics/abc/php.rs +++ b/src/metrics/abc/php.rs @@ -48,7 +48,7 @@ fn php_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { // Whether an arm of `compute` already charges `expr` (already peeled) as // a condition: a ternary (its own arm), a comparison or `instanceof` -// (the token arm), or an `&&` / `||` / `and` / `or` / `xor` chain, whose +// (the token arm), or an `&&` / `||` / `and` / `or` chain, whose // operands each pay through `php_count_condition`. `??` is no condition // token in PHP, so a slot holding one pays for it. fn php_condition_scores_itself(expr: &Node) -> bool { @@ -73,7 +73,6 @@ fn php_condition_scores_itself(expr: &Node) -> bool { | PIPEPIPE | And | Or - | Xor ) }), _ => false, @@ -82,7 +81,7 @@ fn php_condition_scores_itself(expr: &Node) -> bool { // Scores one boolean slot — an `if` / `elseif` / `while` / `do` / `for` // condition, a ternary condition, an operand of an `&&` / `||` / `and` / -// `or` / `xor` chain (see `count_boolean_slot`). PHP is truthy-valued, +// `or` chain (see `count_boolean_slot`). PHP is truthy-valued, // so `if (-$x)`, `if ($x + 1)`, `if ($y = $x)`, `if (A::B)` and // `if ($a ?? $b)` are each a decision, and each scored 0 while the slot // paid only for a fixed list of terminal kinds (#1526). @@ -222,15 +221,17 @@ impl Abc for PhpCode { stats.conditions += 1.; } // Fitzpatrick Rule 9: each operand of a `&&` / `||` / `and` - // / `or` / `xor` chain is one condition (issue #403). PHP - // exposes both the punctuation forms (`&&`, `||`) and the - // low-precedence keyword forms (`and`, `or`, `xor`) as + // / `or` chain is one condition (issue #403). `xor` is no + // chain: it evaluates both operands, so like `^` it is a + // value and its slot pays once (#1536). PHP exposes both + // the punctuation forms (`&&`, `||`) and the + // low-precedence keyword forms (`and`, `or`) as // distinct tokens inside `binary_expression`; both fire // the walker so `connect() or die();`-style idiom counts // the same as `connect() || die();`. `$a && $b || $c` is a // left-nested chain of `binary_expression`s, so an operand // that is itself a chain is paid by its own operator's visit. - AMPAMP | PIPEPIPE | And | Or | Xor => { + AMPAMP | PIPEPIPE | And | Or => { if let Some(chain) = ancestors .parent(node) .filter(|p| p.kind_id() == BinaryExpression) diff --git a/src/metrics/abc/slot_pays_tests.rs b/src/metrics/abc/slot_pays_tests.rs index 6e39403c..8e3575cf 100644 --- a/src/metrics/abc/slot_pays_tests.rs +++ b/src/metrics/abc/slot_pays_tests.rs @@ -787,7 +787,8 @@ fn php_slots_pay_for_any_predicate() { ("$a && $b", 2, 2), ("$a && -$x", 2, 2), ("$a and -$x", 2, 2), - ("$a xor -$x", 2, 2), + // `xor` is a value, as `^` is: one slot, no chain (#1536). + ("$a xor -$x", 1, 1), ("$a || ($y = $x > 1)", 2, 2), // A ternary scores itself through its own arm. ("$a ? $b : $x", 2, 2), @@ -996,6 +997,10 @@ fn perl_slots_pay_for_any_predicate() { // two-operand `unary_expression`; it scored 0. ("$a and $b", 2, 2), ("$a and -$x", 2, 2), + // `xor` is a value, as `^` is: one slot, no chain (#1536). + ("$a ^ $b", 1, 1), + ("$a xor $b", 1, 1), + ("$a xor -$x", 1, 1), ], ); for slot in [ diff --git a/src/metrics/cognitive.rs b/src/metrics/cognitive.rs index b1764322..59229634 100644 --- a/src/metrics/cognitive.rs +++ b/src/metrics/cognitive.rs @@ -7942,21 +7942,27 @@ end", #[cfg(feature = "php")] #[test] - fn php_word_form_xor_forms_boolean_sequence_230() { - check_metrics::( - "( + &format!( + " { // PHP's null-coalescing `??` short-circuits like `&&` / - // `||` and the word-form `and` / `or` / `xor`, so it - // forms boolean sequences alongside them. Mirrors the - // PHP cyclomatic operator set minus the assignment - // form `??=`, which is not a `BinaryExpression`. The word - // forms `and` / `or` are keyed to their symbols; `xor` - // and `??` are operations of their own (#1530). + // `||` and the word-form `and` / `or`, so it forms + // boolean sequences alongside them. Mirrors the PHP + // cyclomatic operator set minus the assignment form + // `??=`, which is not a `BinaryExpression`. The word + // forms `and` / `or` are keyed to their symbols; `??` is + // an operation of its own (#1530). `xor` evaluates both + // operands, so like `^` it is a value, not a sequence + // (#1536). compute_booleans_keyed(node, stats, |id| match id.into() { AMPAMP | And => Some(AMPAMP as u16), PIPEPIPE | Or => Some(PIPEPIPE as u16), - Xor | QMARKQMARK => Some(id), + QMARKQMARK => Some(id), _ => None, }); } diff --git a/src/metrics/cognitive/word_operator_tests.rs b/src/metrics/cognitive/word_operator_tests.rs index 84ca2d66..2c62cc8a 100644 --- a/src/metrics/cognitive/word_operator_tests.rs +++ b/src/metrics/cognitive/word_operator_tests.rs @@ -11,8 +11,10 @@ //! //! The rows that switch operation (`a || b and c`) stay at two //! sequences: precedence decides which operator nests under which, and -//! the walk sees that through the parse. `xor`, `??` and `//` are -//! operations of their own and must not join an `&&` / `||` sequence. +//! the walk sees that through the parse. `??` and `//` are operations of +//! their own and must not join an `&&` / `||` sequence. `xor` evaluates +//! both operands, so it is a value like `^` and no sequence at all +//! (#1536). use std::cell::Cell; @@ -85,10 +87,10 @@ fn php_word_operators_continue_their_symbol_sequence() { ("$a && $b && $c", "$a and $b && $c", 1, 1), ("$a || $b || $c", "$a || $b or $c", 1, 1), ("($a || $b) && $c", "$a || $b and $c", 1, 2), - // `xor` is an operation of its own: it neither joins nor is - // joined by a conjunction. - ("$a ?? $b && $c", "$a xor $b && $c", 1, 2), - ("$a ?? $b ?? $c", "$a xor $b xor $c", 2, 1), + // `xor` is a value, as `^` is (#1536): it binds looser than + // `&&`, so only the conjunction is a sequence. + ("$a ^ ($b && $c)", "$a xor $b && $c", 1, 1), + ("$a ^ $b ^ $c", "$a xor $b xor $c", 2, 0), ]; assert_rows::("foo.php", asg, &words, &rows); } diff --git a/src/metrics/cyclomatic/php.rs b/src/metrics/cyclomatic/php.rs index b769ace3..78dc250f 100644 --- a/src/metrics/cyclomatic/php.rs +++ b/src/metrics/cyclomatic/php.rs @@ -27,7 +27,9 @@ impl Cyclomatic for PhpCode { SwitchStatement | MatchExpression => { stats.cyclomatic_modified += 1.; } - // Both standard and modified. + // Both standard and modified. `xor` is absent: it evaluates + // both operands, so like `^` it is a value, not a decision + // (#1536). IfStatement | ElseIfClause | ElseIfClause2 @@ -41,7 +43,6 @@ impl Cyclomatic for PhpCode { | PIPEPIPE | And | Or - | Xor | QMARKQMARK | QMARKQMARKEQ // Nullsafe operator `?->` (`QMARKDASHGT`) is short-circuit — it diff --git a/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__perl_short_circuit_not_counted_directly_ternary_counts.snap b/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__perl_short_circuit_not_counted_directly_ternary_counts.snap index d4816ee7..d28af22b 100644 --- a/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__perl_short_circuit_not_counted_directly_ternary_counts.snap +++ b/src/metrics/snapshots/big_code_analysis__metrics__abc__tests__perl_short_circuit_not_counted_directly_ternary_counts.snap @@ -5,16 +5,16 @@ expression: metric.abc { "assignments": 7, "branches": 0, - "conditions": 14, - "magnitude": 15.652475842498529, + "conditions": 12, + "magnitude": 13.892443989449804, "value": 0.0, "assignments_average": 3.5, "branches_average": 0.0, - "conditions_average": 7.0, + "conditions_average": 6.0, "assignments_min": 0, "assignments_max": 7, "branches_min": 0, "branches_max": 0, "conditions_min": 0, - "conditions_max": 14 + "conditions_max": 12 } From c36e5ee69a55c93ee8c8d02518e53c9cc4ca7036 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:50:39 -0700 Subject: [PATCH 10/48] fix(bash): score -a/-o and = like their && twins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `[ X -a Y ]`, `[ X ] && [ Y ]` and `[[ X && Y ]]` disagreed. ABC counted the `-a` / `-o` connective as a comparison and never counted a `=` comparison, cyclomatic and cognitive ignored `-a` / `-o`, so `[ "$a" = 1 -a "$b" = 2 ]` scored 2 / 2 / 1 where `[[ "$a" == 1 && "$b" == 2 ]]` scores 3 / 3 / 2. lang_helpers::bash names the two cases: `bash_test_connective` reads a binary `test_operator`'s text (the unary `-a FILE` stays a file test) and `bash_eq_is_comparison` climbs out of the expression tree to a `test_command`, so `(( x = 1 ))` stays an assignment. Cognitive keys `-a` / `-o` to `&&` / `||`, and widens the sequence span over the comparison the grammar wraps around a connective (`"$a" = (1 -o "$b") = …`), which also fixes `[[ "$a" = 1 || "$b" = 2 || "$c" = 3 ]]` scoring two sequences. A bare `test X -a Y` stays out of reach. Fixes #1536 --- big-code-analysis-ast/src/lang_helpers.rs | 1 + .../src/lang_helpers/bash.rs | 57 ++++++ big-code-analysis-book/src/metrics.md | 2 +- src/metrics/abc/bash.rs | 21 ++- src/metrics/bash_test_connective_tests.rs | 162 ++++++++++++++++++ src/metrics/cognitive.rs | 20 ++- src/metrics/cognitive/bash.rs | 41 ++++- src/metrics/cyclomatic/bash.rs | 13 +- src/metrics/mod.rs | 3 + 9 files changed, 311 insertions(+), 9 deletions(-) create mode 100644 big-code-analysis-ast/src/lang_helpers/bash.rs create mode 100644 src/metrics/bash_test_connective_tests.rs diff --git a/big-code-analysis-ast/src/lang_helpers.rs b/big-code-analysis-ast/src/lang_helpers.rs index d23d4685..9d3c6444 100644 --- a/big-code-analysis-ast/src/lang_helpers.rs +++ b/big-code-analysis-ast/src/lang_helpers.rs @@ -13,6 +13,7 @@ //! a helper *from* `metrics::cognitive` would make the parse layer depend //! on the metric layer, the inversion #1376 exists to remove. +pub mod bash; pub mod elixir; // Crate-private: this dialect's only helper is a kind table the three // classifiers in this crate share, and nothing outside names it. diff --git a/big-code-analysis-ast/src/lang_helpers/bash.rs b/big-code-analysis-ast/src/lang_helpers/bash.rs new file mode 100644 index 00000000..f0328d00 --- /dev/null +++ b/big-code-analysis-ast/src/lang_helpers/bash.rs @@ -0,0 +1,57 @@ +//! Bash: the logical connectives and the string comparison of a +//! `[ … ]` / `[[ … ]]` test. + +use crate::Bash; +use crate::node::{Ancestors, Node}; + +/// The `&&` / `||` a `[ … ]` connective stands for: `Some(AMPAMP)` for +/// `-a` and `Some(PIPEPIPE)` for `-o`, when `op`, a child of `parent`, +/// is one. +/// +/// tree-sitter-bash gives every `-x` operator the one kind +/// `test_operator`, so the text decides. Only a binary position joins +/// two tests: the unary `-a FILE` (a file-exists test) has a +/// `unary_expression` parent and is a comparison like `-f FILE`. Every +/// expression kind is matched by name, not by `kind_id`, because the +/// grammar aliases `binary_expression` five times (grammar-dispatch §1). +/// +/// The `test` builtin spelling (`test "$a" = 1 -a "$b" = 2`) is out of +/// reach: the grammar parses its arguments as plain `word`s, so neither +/// its comparisons nor its connectives exist as nodes, and it scores as +/// one command. +#[must_use] +pub fn bash_test_connective(op: &Node, parent: &Node, code: &[u8]) -> Option { + if op.kind_id() != Bash::TestOperator as u16 || parent.kind() != "binary_expression" { + return None; + } + match code.get(op.start_byte()..op.end_byte())? { + b"-a" => Some(Bash::AMPAMP), + b"-o" => Some(Bash::PIPEPIPE), + _ => None, + } +} + +/// Whether `eq`, a Bash `=` token, is the string comparison of a +/// `[ … ]` / `[[ … ]]` test rather than an assignment. +/// +/// Both spellings are a `binary_expression`: `[ "$a" = 1 ]` compares +/// and `(( x = 1 ))` assigns. Climbing out of the test's expression +/// tree must therefore reach the `test_command` itself. The climb +/// passes through `binary_expression` because tree-sitter-bash nests +/// `=` wrongly — `[ "$a" = 1 -a "$b" = 2 ]` parses as +/// `"$a" = (1 -a "$b") = 2` — and through the unary and parenthesized +/// forms (`[[ ! ( "$a" = 1 ) ]]`). A `variable_assignment`'s `=` has no +/// expression parent, so the climb stops at once there. +#[must_use] +pub fn bash_eq_is_comparison<'a>(eq: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + ancestors + .iter(eq) + .map(|(ancestor, _)| ancestor) + .find(|ancestor| { + !matches!( + ancestor.kind(), + "binary_expression" | "unary_expression" | "parenthesized_expression" + ) + }) + .is_some_and(|ancestor| ancestor.kind_id() == Bash::TestCommand as u16) +} diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index cc8d449a..b1ddb1c6 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -152,7 +152,7 @@ application would over-count. | `for` header condition slot: Java, Groovy, C#, C, C++, Objective-C, Mozcpp, JavaScript, TypeScript, TSX, Mozjs, PHP, Go, Perl | `for (; a; )` counts one condition; `for (;;)` counts none | The loop header's condition is a decision-bearing slot like an `if` predicate, so `for (; a; )`, `for (; !a; )` and `for (; (a); )` each report one. A comparison-shaped condition (`i < n`) is already counted by its own operator arm and is not double-counted. Every one of these grammars except Go exposes a `condition` field on `for_statement` (Perl's sits on its C-style `for_statement_1`), so the slot is read by field name rather than by child index; Go's field sits on the nested `for_clause`, and its outer header slot is located structurally (first named child that is neither the body nor a comment). An **empty** condition counts zero in every language: Fitzpatrick's rules count conditional operators and unary conditions that are present, and an omitted test is not a decision. Java and Groovy previously scored `for (;;)` as one vacuously-true condition and no longer do. Tcl, iRules and Bash are command-dispatched grammars whose loop headers carry no boolean-expression slot to route (grammar-dispatch §9) (#1276). | | Ruby | Bare-predicate `if` / `unless` / `while` / `until` (block and modifier forms) count one condition | Idiomatic Ruby favours bare predicates (`if flag`, `x if flag`); counting the condition slot keeps ABC conditions at or above Ruby's cyclomatic decision count (the alignment enforced across the other languages). A comparison (`if a == b`) or `&&` / `\|\|` chain in the predicate is counted by its own operator / walker arm and is not double-counted. Each `&&` / `\|\|` / `and` / `or` operand is scored by the same rule — once, unless its own arm already counts it — so `a && self`, `a && Foo::Bar` and `a && -b` score 2 like `a && b`, wherever the chain is written (#1529). | | Python, JavaScript, TypeScript, TSX, Mozjs, C, C++, Objective-C, Mozcpp, Java, C#, Rust, PHP, Perl, Lua, Groovy, Kotlin, Tcl, iRules | A boolean slot pays one condition unless something inside its predicate already counts it | Every `if` / `while` / `do` / `for` condition, ternary condition, guard and `&&` / `\|\|` operand scores the way Ruby's has since #1520: one condition, unless the predicate — seen through parentheses, a negation, a cast or a plain assignment — is a comparison, chain, ternary or type test that its own rule counts. Before #1526 these slots paid only for a predicate from a fixed list of operand kinds, so `if (-x)`, `if (x + 1)`, `if (this)`, `if (*p)`, `if (flags & k)`, `if (y = x)` and `if (sizeof x)` each scored 0 against a cyclomatic decision of 1, as did Java's, C#'s and Rust's `if (a & b)` and a boolean `switch` / `match` expression. The same rule stops a wrapped comparison paying twice: `if ((int)(x > 1))`, Python's `if (y := x > 1):` and Python's `if not (x > 0):` score 1, not 2, as Java's `if (!(x > 0))` always has. Python's `elif`, PHP's `elseif`, Lua's `elseif` and Perl's `elsif` now score like Ruby's `elsif` and Java's `else if`: one for the `else` plus the predicate slot, so `elif b` no longer scores one below `elif x > 0`. Kotlin joined in #1533, so the slot pays for a `when` it holds, as it does for Java's `switch`, C#'s `switch` expression and Rust's `match`: `if (when (a) { else -> true })` scores 1 (it scored 0) and `if (when (x) { 1 -> true; else -> false })` 2 (it scored 1), its entry paying 1 and the slot 1. Go still pays a slot through an operand list (#1526). | -| Bash | `if` / `elif` / `while` and each non-wildcard `case` arm count one condition | A Bash predicate is a *command*, so the branch keyword itself — not an embedded boolean expression — is the condition signal. Each matches a Bash cyclomatic decision; the bare `*)` case arm (the analogue of `default:`) is excluded, mirroring the cyclomatic standard count. The arithmetic ternary `$(( a ? b : c ))` therefore contributes nothing: it carries no branch keyword, so it falls outside the rule set rather than through a gap in it. | +| Bash | `if` / `elif` / `while` and each non-wildcard `case` arm count one condition | A Bash predicate is a *command*, so the branch keyword itself — not an embedded boolean expression — is the condition signal. Each matches a Bash cyclomatic decision; the bare `*)` case arm (the analogue of `default:`) is excluded, mirroring the cyclomatic standard count. The arithmetic ternary `$(( a ? b : c ))` counts one, as the C-family ternary does (#1268). Each comparison inside a test pays one more: `==`, `!=`, the `-eq` / `-z` / `-f` family, and `=`, which compares in `[ … ]` / `[[ … ]]` and assigns in `(( … ))`. The `[ … ]` connectives `-a` / `-o` join two tests as `&&` / `\|\|` do, and pay nothing, so `[ X -a Y ]`, `[ X ] && [ Y ]` and `[[ X && Y ]]` score alike in every decision metric; a bare `test X -a Y` is all plain words to the grammar and scores none of them (#1536). | | Kotlin | `try` counts a condition alongside `catch` | Fitzpatrick counts both keywords, and Java / C# / C++ / Groovy already count both; Kotlin previously counted only the catch block. | | C#, Java, Rust, Python, Ruby, Elixir | A pattern-match guard is a condition slot, scoring one however it is spelled | A guard is a branch: the pattern can match while the guard fails. The guard's expression is scored exactly as an `if` condition is — one for a call, type test, attribute or bare identifier, one via the operator arm for a comparison — so every spelling agrees, and a compound guard (`when a > 1 && b < 2`) keeps its sub-structure rather than collapsing to one. Before this, ABC scored whatever operator happened to sit inside, so `when x > 2` counted one and the equivalent `when IsEven(x)` counted none. The spellings, per language: C# `when_clause` on a switch arm or `case` section and `catch_filter_clause` on a `catch` (#1422); Java 21's `guard` on a pattern-switch label, in both the arrow and colon forms; Rust's `match_pattern` guard; Python's `case … if g:`; Ruby's `if_guard` / `unless_guard` on a `case … in` arm; Elixir's `when` operator on a `stab_clause` head or a `def` / `defguard` head (#1454). The same change made the guard a **cyclomatic** decision wherever it was not already one — Java, Ruby and Elixir — and excluded Elixir's typespec `when` (`@spec f(a) :: a when a: integer`), which spells the same token as a guard but is type syntax. Groovy and Kotlin are absent because neither pinned grammar has a guard production at all: Kotlin 2.1 guard syntax does not parse, so there is nothing to classify (#1454). | | Kotlin | A subject-less `when` arm scores through the predicate slot; a subject-ful arm scores per entry | A subject-less arm's condition (`when { x > 5 -> … }`) is an ordinary boolean expression, compiled as an `if` predicate, so the comparison inside it is already counted by the operator arms and a blanket per-entry increment double-counted it. A subject-ful arm (`when (x) { in 1..2 -> … }`) lists a pattern rather than an independent boolean expression, so the implicit `subject == pattern` is a decision nothing in the source spells and the entry itself pays for it (#1421). | diff --git a/src/metrics/abc/bash.rs b/src/metrics/abc/bash.rs index d99e1079..dd08dd26 100644 --- a/src/metrics/abc/bash.rs +++ b/src/metrics/abc/bash.rs @@ -10,6 +10,7 @@ )] use super::{Abc, Stats}; +use crate::lang_helpers::bash::{bash_eq_is_comparison, bash_test_connective}; use crate::*; impl Abc for BashCode { @@ -36,8 +37,11 @@ impl Abc for BashCode { // Three condition signals share this arm: // // - Comparison operators inside `[[ … ]]` and `(( … ))`, plus - // the prefix test operators `-z`, `-n`, `-eq`, `-lt`, … which - // the grammar emits as `Bash::TestOperator`. + // the test operators `-z`, `-n`, `-eq`, `-lt`, … which the + // grammar emits as `Bash::TestOperator`. The `-a` / `-o` + // connectives share that kind and are excluded: they join + // two tests the way `&&` / `||` do, and neither pays + // (#1536). // - Control-flow branches (`if`/`elif`/`while`). A Bash predicate // is a command, so the branch keyword itself is the only // condition signal. These branch keywords mirror the matching @@ -53,7 +57,6 @@ impl Abc for BashCode { | Bash::LTEQ | Bash::GTEQ | Bash::EQTILDE - | Bash::TestOperator | Bash::IfStatement | Bash::ElifClause | Bash::WhileStatement @@ -61,6 +64,18 @@ impl Abc for BashCode { | Bash::TernaryExpression2 => { stats.conditions += 1.; } + Bash::TestOperator + if ancestors + .parent(node) + .is_none_or(|parent| bash_test_connective(node, &parent, code).is_none()) => + { + stats.conditions += 1.; + } + // `=` compares strings inside `[ … ]` / `[[ … ]]`, exactly as + // `==` does, and assigns inside `(( … ))` (#1536). + Bash::EQ if bash_eq_is_comparison(node, ancestors) => { + stats.conditions += 1.; + } // `<` and `>` are comparisons only inside a `binary_expression`. // The same two tokens spell an I/O redirection (`cmd > out`, // `read x < in`), which the grammar parents under diff --git a/src/metrics/bash_test_connective_tests.rs b/src/metrics/bash_test_connective_tests.rs new file mode 100644 index 00000000..2bbbf732 --- /dev/null +++ b/src/metrics/bash_test_connective_tests.rs @@ -0,0 +1,162 @@ +//! A `[ … ]` test scores like its `&&` / `||` twins (#1536). +//! +//! Bash joins two tests three ways: `[ X -a Y ]`, `[ X ] && [ Y ]` and +//! `[[ X && Y ]]` (and `-o` / `||` likewise). Before #1536 the first +//! scored no cyclomatic or cognitive decision for its `-a`, ABC counted +//! that `-a` as a comparison, and a `=` comparison scored no ABC +//! condition in any spelling. Every row groups the three spellings, +//! and each must score the row's decisions. + +use std::cell::Cell; + +use crate::test_support::check_func_space_only; +use crate::*; + +/// `[cyclomatic, abc conditions, cognitive]` sums of the root space. +/// The cyclomatic sum carries the unit's and the function's base of 1 +/// each. Bash ABC pays one condition for the `if` and one for each +/// comparison, and nothing for a connective. +#[cfg(feature = "bash")] +type Decisions = [u64; 3]; + +#[cfg(feature = "bash")] +fn measure(body: &str) -> Decisions { + let out = Cell::new([0; 3]); + check_func_space_only::( + &format!("f() {{\n {body}\n}}\n"), + "foo.sh", + &[Metric::Cyclomatic, Metric::Abc, Metric::Cognitive], + |space| { + let m = &space.metrics; + out.set([ + m.cyclomatic.cyclomatic_sum(), + m.abc.conditions_sum(), + m.cognitive.cognitive_sum(), + ]); + }, + ); + out.get() +} + +#[cfg(feature = "bash")] +#[test] +fn bash_test_connectives_score_like_their_twins_1536() { + // `(single_bracket, chained_commands, double_bracket, decisions)`. + let rows: [(&str, &str, &str, Decisions); 8] = [ + ( + r#"if [ "$a" = 1 -a "$b" = 2 ]; then :; fi"#, + r#"if [ "$a" = 1 ] && [ "$b" = 2 ]; then :; fi"#, + r#"if [[ "$a" = 1 && "$b" = 2 ]]; then :; fi"#, + [4, 3, 2], + ), + ( + r#"if [ "$a" == 1 -a "$b" == 2 ]; then :; fi"#, + r#"if [ "$a" == 1 ] && [ "$b" == 2 ]; then :; fi"#, + r#"if [[ "$a" == 1 && "$b" == 2 ]]; then :; fi"#, + [4, 3, 2], + ), + ( + r#"if [ "$a" -eq 1 -a "$b" -eq 2 ]; then :; fi"#, + r#"if [ "$a" -eq 1 ] && [ "$b" -eq 2 ]; then :; fi"#, + r#"if [[ "$a" -eq 1 && "$b" -eq 2 ]]; then :; fi"#, + [4, 3, 2], + ), + ( + r#"if [ "$a" = 1 -o "$b" = 2 ]; then :; fi"#, + r#"if [ "$a" = 1 ] || [ "$b" = 2 ]; then :; fi"#, + r#"if [[ "$a" = 1 || "$b" = 2 ]]; then :; fi"#, + [4, 3, 2], + ), + // A switch of operation is a second cognitive sequence in every + // spelling. + ( + r#"if [ "$a" = 1 -a "$b" = 2 -o "$c" = 3 ]; then :; fi"#, + r#"if [ "$a" = 1 ] && [ "$b" = 2 ] || [ "$c" = 3 ]; then :; fi"#, + r#"if [[ "$a" = 1 && "$b" = 2 || "$c" = 3 ]]; then :; fi"#, + [5, 4, 3], + ), + // `-a` and `-o` key to the symbols they stand for: a run of one + // is a single sequence. The grammar nests `=`, `==` and `!=` + // around the connectives (`"$a" = (1 -o "$b") = …`), so these + // rows also pin `bash_sequence_end`; `[[ … = … || … ]]` scored 3 + // before #1536 for the same reason. + ( + r#"if [ "$a" = 1 -o "$b" = 2 -o "$c" = 3 ]; then :; fi"#, + r#"if [ "$a" = 1 ] || [ "$b" = 2 ] || [ "$c" = 3 ]; then :; fi"#, + r#"if [[ "$a" = 1 || "$b" = 2 || "$c" = 3 ]]; then :; fi"#, + [5, 4, 2], + ), + ( + r#"if [ "$a" == 1 -o "$b" == 2 -o "$c" == 3 ]; then :; fi"#, + r#"if [ "$a" == 1 ] || [ "$b" == 2 ] || [ "$c" == 3 ]; then :; fi"#, + r#"if [[ "$a" == 1 || "$b" == 2 || "$c" == 3 ]]; then :; fi"#, + [5, 4, 2], + ), + ( + r#"if [ "$a" != 1 -o "$b" != 2 -o "$c" != 3 ]; then :; fi"#, + r#"if [ "$a" != 1 ] || [ "$b" != 2 ] || [ "$c" != 3 ]; then :; fi"#, + r#"if [[ "$a" != 1 || "$b" != 2 || "$c" != 3 ]]; then :; fi"#, + [5, 4, 2], + ), + ]; + for (single, chained, double, decisions) in rows { + assert!( + single.contains(" -a ") || single.contains(" -o "), + "`{single}` lost its connective" + ); + for body in [single, chained, double] { + assert_eq!(measure(body), decisions, "`{body}`"); + } + } +} + +/// What the gates must leave alone: a unary `-a FILE` is a file test +/// like `-f FILE`, a `=` keeps comparing however the test nests it, `=` +/// inside `(( … ))` is an assignment, and a correctly nested switch of +/// operation is still three sequences. +#[cfg(feature = "bash")] +#[test] +fn bash_test_connective_gates_keep_their_controls_1536() { + // `(body, twin, decisions)`; `twin` spells the same test another + // way and must agree. + let rows: [(&str, &str, Decisions); 6] = [ + ( + "if [ -a x ]; then :; fi", + "if [ -f x ]; then :; fi", + [3, 2, 1], + ), + ( + r#"if [ "$a" = 1 ]; then :; fi"#, + r#"if [ "$a" == 1 ]; then :; fi"#, + [3, 2, 1], + ), + ( + r#"if [[ ( "$a" = 1 ) ]]; then :; fi"#, + r#"if [[ ( "$a" == 1 ) ]]; then :; fi"#, + [3, 2, 1], + ), + ( + r#"if [[ ! ( "$a" = 1 ) ]]; then :; fi"#, + r#"if [[ ! ( "$a" == 1 ) ]]; then :; fi"#, + [3, 2, 1], + ), + ("(( x = 1 ))", "x=1", [2, 0, 0]), + ( + r#"if [[ ( "$a" = 1 || "$b" = 2 ) && ( "$c" = 3 || "$d" = 4 ) ]]; then :; fi"#, + r#"if [[ ( "$a" == 1 || "$b" == 2 ) && ( "$c" == 3 || "$d" == 4 ) ]]; then :; fi"#, + [6, 5, 4], + ), + ]; + for (body, twin, decisions) in rows { + assert_eq!( + measure(twin), + decisions, + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure(body), + decisions, + "`{body}` must score like `{twin}`" + ); + } +} diff --git a/src/metrics/cognitive.rs b/src/metrics/cognitive.rs index 59229634..762c625f 100644 --- a/src/metrics/cognitive.rs +++ b/src/metrics/cognitive.rs @@ -217,9 +217,25 @@ fn compute_booleans_keyed Option>( stats: &mut Stats, sequence_key: F, ) { - let enclosing_end = node.end_byte(); + compute_booleans_by_node(node, node.end_byte(), stats, |child| { + sequence_key(child.kind_id()) + }); +} + +/// [`compute_booleans_keyed`] for an operator whose kind alone does not +/// say which operation it is — Bash's `-a` / `-o` share `test_operator` +/// with every comparison, so the key reads the child itself — and for a +/// sequence whose extent is not `node`'s own span. `enclosing_end` is +/// where the sequence `node` belongs to ends: an operator inside it +/// continues a like operator seen earlier in it. +fn compute_booleans_by_node Option>( + node: &Node, + enclosing_end: usize, + stats: &mut Stats, + sequence_key: F, +) { for child in node.children() { - if let Some(key) = sequence_key(child.kind_id()) { + if let Some(key) = sequence_key(&child) { stats.structural = stats .boolean_seq diff --git a/src/metrics/cognitive/bash.rs b/src/metrics/cognitive/bash.rs index 82a05d43..5ab6f576 100644 --- a/src/metrics/cognitive/bash.rs +++ b/src/metrics/cognitive/bash.rs @@ -12,11 +12,38 @@ )] use super::*; +use crate::lang_helpers::bash::bash_test_connective; + +// FIXME(#1536 upstream): tree-sitter-bash 0.25.1 lets a test's +// comparison swallow the connectives around it, so +// `[ "$a" = 1 -o "$b" = 2 -o "$c" = 3 ]` parses as +// `"$a" = (1 -o "$b") = (2 -o "$c") = 3` (and `[[ … = … || … ]]` the +// same). The two `-o` then sit in sibling subtrees, neither inside the +// other's span, and the run of one operation scored two sequences. A +// connective's expression is an operand of a comparison only in that +// mis-nested tree, so the sequence it belongs to ends where the +// outermost such comparison does. A correct tree (`==` inside `[[ … ]]`, +// `-eq`) never puts a connective under a comparison, so this returns +// `node`'s own end there. The mis-nested tree has lost precedence, so a +// mixed run is scored in textual order: `[[ "$a" = 1 || "$b" = 2 && +// "$c" = 3 || "$d" = 4 ]]` scores three sequences where the `==` twin, +// nested `(a || (b && c)) || d`, scores two. +fn bash_sequence_end(node: &Node, ancestors: Ancestors) -> usize { + ancestors + .iter(node) + .map(|(ancestor, _)| ancestor) + .take_while(|ancestor| { + ancestor.kind_id() == Bash::BinaryExpression3 as u16 + && ancestor.wraps_any(&[Bash::EQ as u16, Bash::EQEQ as u16, Bash::BANGEQ as u16]) + }) + .last() + .map_or(node.end_byte(), |outermost| outermost.end_byte()) +} impl Cognitive for BashCode { fn compute<'a>( node: &Node<'a>, - _code: &'a [u8], + code: &'a [u8], ancestors: Ancestors<'a, '_>, stats: &mut Stats, nesting_map: &mut NestingMap, @@ -53,9 +80,19 @@ impl Cognitive for BashCode { // parenthesized sub-expressions). Verified empirically // against tree-sitter-bash 0.25.1 — the other four // `BinaryExpression*` enum variants never wrap `&&` / `||`. - List | BinaryExpression3 => { + // Inside `[ … ]` the same expression joins two tests with + // `-a` / `-o`, keyed to the symbol it stands for so the + // three spellings score alike (#1536). + List => { compute_booleans(node, stats, AMPAMP, PIPEPIPE); } + BinaryExpression3 => { + let end = bash_sequence_end(node, ancestors); + compute_booleans_by_node(node, end, stats, |child| match child.kind_id().into() { + AMPAMP | PIPEPIPE => Some(child.kind_id()), + _ => bash_test_connective(child, node, code).map(|op| op as u16), + }); + } FunctionDefinition => { enter_function_boundary(&mut nesting, node, ancestors, &[FunctionDefinition]); } diff --git a/src/metrics/cyclomatic/bash.rs b/src/metrics/cyclomatic/bash.rs index ebfeda70..3fc84bad 100644 --- a/src/metrics/cyclomatic/bash.rs +++ b/src/metrics/cyclomatic/bash.rs @@ -7,12 +7,13 @@ )] use super::*; +use crate::lang_helpers::bash::bash_test_connective; impl Cyclomatic for BashCode { fn compute<'a>( node: &Node<'a>, code: &'a [u8], - _ancestors: Ancestors<'a, '_>, + ancestors: Ancestors<'a, '_>, stats: &mut Stats, ) { match node.kind_id().into() { @@ -51,6 +52,16 @@ impl Cyclomatic for BashCode { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } + // `[ … ]` spells `&&` / `||` as `-a` / `-o`, a decision each + // like the `[ … ] && [ … ]` and `[[ … && … ]]` twins (#1536). + Bash::TestOperator + if ancestors + .parent(node) + .is_some_and(|parent| bash_test_connective(node, &parent, code).is_some()) => + { + stats.cyclomatic += 1.; + stats.cyclomatic_modified += 1.; + } _ => {} } } diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index c397e0b3..8ee7b9b3 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -42,6 +42,9 @@ pub mod wmc; // enabling exactly the languages it exists to cover // (`.claude/rules/testing.md`, #1286). #[cfg(test)] +#[path = "bash_test_connective_tests.rs"] +mod bash_test_connective_tests; +#[cfg(test)] #[cfg(any( feature = "bash", feature = "c", From 0ffc675aef783dec2f9ffb28a5604d2f469277f9 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:02:19 -0700 Subject: [PATCH 11/48] fix(elixir): bill a named operator once `&==/2`, `&and/2` and `Kernel.||(a, b)` name an operator without applying it, and the grammar wraps the token in an `operator_identifier`. Halstead billed the wrapper as an operand and the token inside it as an operator, so `a = &==/2` scored n1 4 where `a = &foo/2` scores 3. Cyclomatic counted a named `&&` / `||` / `and` / `or` as a decision in both tiers. The getter now returns Unknown for any token whose parent is an `operator_identifier`, keeping the wrapper (the only node a childless `..` has). Cyclomatic gates its short-circuit arm on an applying `binary_operator`, through `lang_helpers::elixir:: elixir_applying_operator`, which ABC now shares. Fixes #1534 --- .bca-baseline.toml | 2 +- big-code-analysis-ast/src/getter/elixir.rs | 11 ++ .../src/lang_helpers/elixir.rs | 27 +++- big-code-analysis-book/src/metrics.md | 10 +- src/metrics/abc/elixir.rs | 20 +-- src/metrics/cyclomatic/elixir.rs | 9 +- src/metrics/elixir_named_operator_tests.rs | 133 ++++++++++++++++++ src/metrics/mod.rs | 3 + 8 files changed, 195 insertions(+), 20 deletions(-) create mode 100644 src/metrics/elixir_named_operator_tests.rs diff --git a/.bca-baseline.toml b/.bca-baseline.toml index a3b52354..0701b103 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -231,7 +231,7 @@ value = 57421.05047463073 path = "big-code-analysis-ast/src/getter/elixir.rs" qualified = "ElixirCode::get_op_type" metric = "halstead.effort" -value = 68425.65670070829 +value = 76992.47971865896 [[entry]] path = "big-code-analysis-ast/src/getter/irules.rs" diff --git a/big-code-analysis-ast/src/getter/elixir.rs b/big-code-analysis-ast/src/getter/elixir.rs index 19b1220f..c436f3a0 100644 --- a/big-code-analysis-ast/src/getter/elixir.rs +++ b/big-code-analysis-ast/src/getter/elixir.rs @@ -142,6 +142,17 @@ impl Getter for ElixirCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Elixir as E; + // A *named* operator (`&==/2`, `&and/2`, `Kernel.||(a, b)`) is one + // `operator_identifier` holding the operator token. The wrapper is + // the operand, as `foo` is in `&foo/2`, so the token inside it must + // not also bill an operator (grammar-dispatch §5, #1534). The + // wrapper is the keeper because it exists for every spelling: `..` + // is a childless `operator_identifier`, so keeping the leaf would + // lose it. + if ancestors.parent_has_kind(node, E::OperatorIdentifier as u16) { + return TokenRole::Unknown; + } + match node.kind_id().into() { // Sigil delimiter punctuation. The delimiter choice is // spelling, not semantics — `~r/abc/` is not two divisions diff --git a/big-code-analysis-ast/src/lang_helpers/elixir.rs b/big-code-analysis-ast/src/lang_helpers/elixir.rs index 2c040e3d..7a200d15 100644 --- a/big-code-analysis-ast/src/lang_helpers/elixir.rs +++ b/big-code-analysis-ast/src/lang_helpers/elixir.rs @@ -1,4 +1,5 @@ -//! Elixir: keyword `Call` identity (`def`, `defmodule`, `quote`, …). +//! Elixir: keyword `Call` identity (`def`, `defmodule`, `quote`, …), +//! and applied versus named operators. use crate::Elixir; use crate::node::{Ancestors, Node}; @@ -86,3 +87,27 @@ pub fn elixir_do_block_call_children<'a>( .flat_map(|do_block| do_block.children()) .filter(|stmt| stmt.kind_id() == Elixir::Call as u16) } + +/// The `binary_operator` that applies `op`, an operator token, or `None` +/// when the token is not applied there. +/// +/// An operator token also *names* the operator: `&==/2`, `&and/2` and +/// `Kernel.||(a, b)` hold it inside an `operator_identifier`, and the +/// single-token `atom` / `keyword` spell it too. None of those compares, +/// chains or decides anything, and under error recovery the parent can +/// be an `ERROR` with no operands, so every decision metric keys on this +/// parent rather than on the token (#1531, #1534). +/// +/// Matched by rule name rather than by `kind_id`: the grammar aliases +/// `binary_operator` to three ids (`Elixir::BinaryOperator` through +/// `BinaryOperator3`), and a name comparison stays correct when a bump +/// adds a fourth (grammar-dispatch §1). +#[must_use] +pub fn elixir_applying_operator<'a>( + op: &Node<'a>, + ancestors: Ancestors<'a, '_>, +) -> Option> { + ancestors + .parent(op) + .filter(|parent| parent.kind() == "binary_operator") +} diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index b1ddb1c6..45c77879 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -594,7 +594,7 @@ basis; the rules deliberately exclude pure layout punctuation like parentheses and statement separators, which is why the Halstead totals are *not* the same as the Tokens count. -Six classification rules are worth knowing because they are choices +Seven classification rules are worth knowing because they are choices rather than consequences, and because several grammars spell the tokens involved the same way they spell real operators: @@ -701,6 +701,14 @@ tokens involved the same way they spell real operators: an operand, and the `ref` in `M(ref int a)` is the same operator as the `ref` in the call `M(ref x)`, adding to `total_operators` but not to `unique_operators` (#1418). +- **A named operator is one operand.** Elixir names an operator + without applying it in a capture (`&==/2`, `&and/2`) or a remote + call (`Kernel.||(a, b)`), and its grammar wraps the token in an + `operator_identifier`. That wrapper is the operand, as `foo` is in + `&foo/2`, so the token inside it adds no operator; `&==/2` scores + what `&foo/2` does. A named `&&` / `||` / `and` / `or` is no + decision either (#1534). This differs from a C# `operator <`, whose + `<` has no wrapper and so stays the operator the rule above makes it. ### Derived metrics diff --git a/src/metrics/abc/elixir.rs b/src/metrics/abc/elixir.rs index fe3dc89c..a13bf622 100644 --- a/src/metrics/abc/elixir.rs +++ b/src/metrics/abc/elixir.rs @@ -10,16 +10,11 @@ )] use super::{Abc, Stats, count_boolean_slot, count_field_operands, last_operand, wrapped_operand}; -use crate::lang_helpers::elixir::elixir_call_keyword; +use crate::lang_helpers::elixir::{elixir_applying_operator, elixir_call_keyword}; use crate::*; -/// The grammar rule an applied binary operator hangs off, as opposed to -/// `operator_identifier`, which is how one is *named* -/// (`& + if elixir_applying_operator(node, ancestors).is_some() => { stats.conditions += 1.; } @@ -450,10 +443,7 @@ impl Abc for ElixirCode { // `binary_operator`s, so an operand that is itself a chain is // paid by its own operator's visit. E::AMPAMP | E::PIPEPIPE | E::And | E::Or => { - if let Some(chain) = ancestors - .parent(node) - .filter(|parent| parent.kind() == BINARY_OPERATOR) - { + if let Some(chain) = elixir_applying_operator(node, ancestors) { count_field_operands(&chain, elixir_count_condition, &mut stats.conditions); } } diff --git a/src/metrics/cyclomatic/elixir.rs b/src/metrics/cyclomatic/elixir.rs index 88eb693b..40108a9f 100644 --- a/src/metrics/cyclomatic/elixir.rs +++ b/src/metrics/cyclomatic/elixir.rs @@ -100,8 +100,13 @@ impl Cyclomatic for ElixirCode { stats.cyclomatic_modified += 1.; } // Short-circuit booleans add a decision point in both - // metrics. - E::AMPAMP | E::PIPEPIPE | E::And | E::Or => { + // metrics — where applied. `&and/2`, `&||/2` and + // `Kernel.||(a, b)` only name the operator and decide + // nothing (#1534); ABC and cognitive gate on the same parent. + E::AMPAMP | E::PIPEPIPE | E::And | E::Or + if crate::lang_helpers::elixir::elixir_applying_operator(node, ancestors) + .is_some() => + { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } diff --git a/src/metrics/elixir_named_operator_tests.rs b/src/metrics/elixir_named_operator_tests.rs new file mode 100644 index 00000000..da0abebf --- /dev/null +++ b/src/metrics/elixir_named_operator_tests.rs @@ -0,0 +1,133 @@ +//! A *named* Elixir operator scores like any other name (#1534). +//! +//! `&==/2`, `&and/2` and `Kernel.||(a, b)` name an operator without +//! applying it: the grammar wraps the token in an `operator_identifier`. +//! Halstead billed both the wrapper (an operand) and the token inside it +//! (an operator), and cyclomatic counted a named `&&` / `||` / `and` / +//! `or` as a decision. Every row pairs the fixture with a twin naming an +//! ordinary function, which is the oracle; the twin's numbers are +//! asserted outright too, so a row cannot pass by both drifting. + +use std::cell::Cell; + +use crate::test_support::check_func_space_only; +use crate::*; + +/// `[n1, N1, n2, N2]`. +#[cfg(feature = "elixir")] +type Halstead = [u64; 4]; + +/// `[cyclomatic, cyclomatic_modified]` sums of the root space, which +/// carries a base of 1 each. +#[cfg(feature = "elixir")] +type Cyclomatic = [u64; 2]; + +#[cfg(feature = "elixir")] +fn measure(source: &str) -> (Halstead, Cyclomatic) { + let out = Cell::new(([0; 4], [0; 2])); + check_func_space_only::( + &format!("{source}\n"), + "foo.exs", + &[Metric::Halstead, Metric::Cyclomatic], + |space| { + let m = &space.metrics; + out.set(( + [ + m.halstead.unique_operators(), + m.halstead.total_operators(), + m.halstead.unique_operands(), + m.halstead.total_operands(), + ], + [ + m.cyclomatic.cyclomatic_sum(), + m.cyclomatic.cyclomatic_modified_sum(), + ], + )); + }, + ); + out.get() +} + +/// Whether `source` parses with an `operator_identifier`, so a row +/// whose fixture drifted to an ordinary name cannot pass by comparing +/// the twin with itself. +#[cfg(feature = "elixir")] +fn names_an_operator(source: &str) -> bool { + let parser = ElixirParser::new( + source.as_bytes().to_vec(), + std::path::Path::new("foo.exs"), + None, + ); + parser + .root() + .preorder() + .any(|node| node.kind_id() == Elixir::OperatorIdentifier as u16) +} + +/// Each named operator bills one operand — its `operator_identifier` — +/// and no operator, exactly as `foo` does in `&foo/2`, and decides +/// nothing. Before #1534 each row billed one operator more, and the +/// `&&` / `||` / `and` / `or` rows scored a decision in both tiers. +#[cfg(feature = "elixir")] +#[test] +fn elixir_named_operator_scores_like_a_named_function() { + // `(twin, named, halstead)`; every row decides nothing. + let rows: [(&str, &str, Halstead); 15] = [ + ("a = &foo/2", "a = &==/2", [3, 3, 3, 3]), + ("a = &foo/2", "a = &in/2", [3, 3, 3, 3]), + ("a = &foo/2", "a = ¬ in/2", [3, 3, 3, 3]), + ("a = &foo/2", "a = &and/2", [3, 3, 3, 3]), + ("a = &foo/2", "a = &or/2", [3, 3, 3, 3]), + ("a = &foo/2", "a = &||/2", [3, 3, 3, 3]), + ("a = &foo/1", "a = ¬/1", [3, 3, 3, 3]), + ("a = &foo/1", "a = &!/1", [3, 3, 3, 3]), + ("a = &foo/2", "a = &+/2", [3, 3, 3, 3]), + ("a = Kernel.foo(x, y)", "a = Kernel.==(x, y)", [4, 4, 5, 5]), + ("a = Kernel.foo(x, y)", "a = Kernel.||(x, y)", [4, 4, 5, 5]), + // `..` is a childless `operator_identifier`: the wrapper is the + // only node that can bill it, which is why it is the keeper. + ("a = foo", "a = ..", [1, 1, 2, 2]), + ("a = &foo/0", "a = &../0", [3, 3, 3, 3]), + // The lexer reads `&&&` as one token, so `&&&/2` holds no + // capture `&`: it is the name `&&&` divided by 2, like `foo/2`. + // `&&/1` lexes the same way, as the name `&&`. + ("a = foo/2", "a = &&&/2", [2, 2, 3, 3]), + ("a = foo/1", "a = &&/1", [2, 2, 3, 3]), + ]; + for (twin, named, halstead) in rows { + assert!( + names_an_operator(named), + "`{named}` lost its operator_identifier" + ); + let want = (halstead, [1, 1]); + assert_eq!(measure(twin), want, "`{twin}` moved; re-derive the row"); + assert_eq!(measure(named), want, "`{named}` must score like `{twin}`"); + } +} + +/// The operand is keyed by the operator's text, so two named operators +/// stay two names: `a`, `==`, `2`, `b`, `<`, with `2` twice. +#[cfg(feature = "elixir")] +#[test] +fn elixir_named_operators_are_distinct_operands() { + let (halstead, _) = measure("a = &==/2\nb = & Date: Fri, 9 Oct 2026 12:32:39 -0700 Subject: [PATCH 12/48] fix(elixir): count with <- clauses, free reduce head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each `pattern <- expr` clause of a `with` either matches or falls through to the `else`, so it is a decision: +1 standard and modified cyclomatic and one ABC condition, like a Rust `if let` or `?`. A `when` guard on that pattern is now a guard, as on a clause. A `for` generator's `<-` and its guard stay non-decisions; the `for` already pays its loop decision. Modified cyclomatic no longer charges `with` a flat +1. Its `else` clauses collapse into one modified decision only when one of them is a decision, so `else _ -> 0` is free in both tiers, as Rust's `if let … else` is. A `for … reduce:` block's accumulator clauses now take the multi-clause `fn` head skip (#776): a single `acc -> …` clause scores nothing, and n clauses score n-1, a trailing `_ ->` included. Fixes #1535 --- .bca-baseline.toml | 6 + big-code-analysis-book/src/metrics.md | 10 +- src/metrics/abc.rs | 9 +- src/metrics/abc/elixir.rs | 13 +- src/metrics/cyclomatic.rs | 170 ++++++++++++++++- src/metrics/cyclomatic/elixir.rs | 42 +++- src/metrics/npa/shared.rs | 263 ++++++++++++++++++++------ 7 files changed, 430 insertions(+), 83 deletions(-) diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 0701b103..6cc5b49b 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -1037,6 +1037,12 @@ qualified = "python_self_attr_name_bytes" metric = "nexits" value = 6.0 +[[entry]] +path = "src/metrics/npa/shared.rs" +qualified = "" +metric = "loc.ploc" +value = 558.0 + [[entry]] path = "src/ops.rs" qualified = "ops_inner" diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 45c77879..8ea1307c 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -158,7 +158,7 @@ application would over-count. | Kotlin | A subject-less `when` arm scores through the predicate slot; a subject-ful arm scores per entry | A subject-less arm's condition (`when { x > 5 -> … }`) is an ordinary boolean expression, compiled as an `if` predicate, so the comparison inside it is already counted by the operator arms and a blanket per-entry increment double-counted it. A subject-ful arm (`when (x) { in 1..2 -> … }`) lists a pattern rather than an independent boolean expression, so the implicit `subject == pattern` is a decision nothing in the source spells and the entry itself pays for it (#1421). | | Ruby | A subject-less `case` scores each `when` pattern as an `\|\|` operand; a subject-ful `case` scores per clause | A subject-less clause (`case; when x > 5`) lists ordinary boolean expressions, evaluated like an `if` predicate, and several patterns (`when a, b`) are an implicit `\|\|`. Each pattern therefore scores once unless its own operator already counted it, so `when x > 5` scores 1 like `if x > 5`, and `when a, b` scores 2 like `if a \|\| b`. Before this, a blanket per-clause increment counted a comparison twice. A subject-ful clause (`case x; when 1`) matches each pattern with an implicit `pattern === x` that nothing in the source spells, so the clause pays once (#1453). | | Go | A tagless `switch` scores each case expression as an `\|\|` operand; a tagged `switch` scores per case | A tagless switch (`switch { case x > 5: }`, with or without an init statement) is `switch true`: each case expression is an ordinary boolean scored by the same slot an `if` predicate uses, and a list (`case a, b:`) is an implicit `\|\|`. So `case x > 5:` scores 1 like `if x > 5`, and `case a, b:` scores 2 like `if a \|\| b`. Before this, a blanket per-case increment counted a comparison twice. A tagged case (`switch x { case 1: }`) compares with an implicit `x == 1` that nothing in the source spells, so the case pays once; type-switch and `select` cases are unchanged (#1523). | -| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive` (its `after` timeout included), of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct, and a `with` with no `else` now scores nothing, level with cyclomatic. The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). | +| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive` (its `after` timeout included), of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). | | C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these `<` / `>` as an operator, and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | | C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | @@ -498,6 +498,14 @@ echoes the long-standing `-m` mode from Terry Yin's many readers will first have seen it.) Both numbers are exported side by side; pick one and be consistent. +Only alternatives collapse. A sequence of tests is still one decision +per test in both variants. Elixir's `with` is the case where the two +meet: each `pattern <- expr` clause is a separate test, scored once in +both variants, as Rust's `if let` and `?` are. The clauses of its +`else` are alternatives, so modified scores them once, and only when +one of them is a decision: `else _ -> 0` selects nothing, like a plain +`else`, and is free in both variants (#1535). + ### Counting Rust's `?` operator By default Rust's `?` operator (the `try_expression` grammar node) diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index 37859660..526a9bb9 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -11711,7 +11711,8 @@ end // scores itself (the `in` arm), `rescue_alias` a sole pattern the // clause slot pays for, `catch_two` a multi-pattern clause, `recv` a // `receive`'s `after` timeout clause, and `with_else` the `else` of - // a `with`, whose clauses pay rather than a Rule 5 `else`. + // a `with`, whose clauses pay rather than a Rule 5 `else`. Its `<-` + // clause pays one more since #1535, in both metrics. #[cfg(feature = "elixir")] #[test] fn elixir_clause_constructs_pay_per_clause() { @@ -11783,7 +11784,7 @@ end ("case2", 2, 3), ("case_default", 2, 3), ("case_guard", 2, 3), - ("with_else", 1, 2), + ("with_else", 2, 3), ("recv", 3, 4), ("rescue_in", 1, 2), ("rescue_alias", 1, 2), @@ -11904,8 +11905,8 @@ end ("rb", 2, 3), ("tk", 1, 2), ("tb", 1, 2), - ("wk", 1, 2), - ("wb", 1, 2), + ("wk", 2, 3), + ("wb", 2, 3), ], ); }); diff --git a/src/metrics/abc/elixir.rs b/src/metrics/abc/elixir.rs index a13bf622..3aa23e78 100644 --- a/src/metrics/abc/elixir.rs +++ b/src/metrics/abc/elixir.rs @@ -254,8 +254,8 @@ impl Abc for ElixirCode { // (`==`, `===`, `!=`, `!==`, `<`, `>`, `<=`, `>=`, `in`, `not in`), // the boolean slots scored by `elixir_count_condition` (`if` / // `unless` predicates, guards, chain operands), one per clause of a - // clause construct (`elixir_count_clause`), and the `else` of an - // `if` / `unless` (Rule 5). + // clause construct (`elixir_count_clause`) and per `<-` clause of a + // `with`, and the `else` of an `if` / `unless` (Rule 5). // `for` / `while` are looping forms — not condition-shaped per // the issue body's literal list — so we omit them. // @@ -450,6 +450,15 @@ impl Abc for ElixirCode { E::StabClause if npa::elixir_clause_is_decision(node, code, ancestors) => { elixir_count_clause(node, &mut stats.conditions); } + // A `with`'s `pattern <- expr` clause is a pattern-match + // condition (Rule 9), the decision `Cyclomatic` counts + // through the same gate (#1535). One flat condition: a + // pattern is never an operator that scores itself, and a + // guard on it (`{:ok, a} when g <- x`) pays its own slot + // through the `when` arm, as a guarded clause's does. + E::LTDASH if npa::elixir_arrow_is_with_clause(node, code, ancestors) => { + stats.conditions += 1.; + } _ => {} } } diff --git a/src/metrics/cyclomatic.rs b/src/metrics/cyclomatic.rs index 0cc59d9a..8111aa1a 100644 --- a/src/metrics/cyclomatic.rs +++ b/src/metrics/cyclomatic.rs @@ -6084,22 +6084,172 @@ f() { } // `with` chains use `<-` arrows, which parse as `binary_operator` - // nodes — NOT `stab_clause`s — so the `with`-head clauses do not - // contribute to standard CCN per-arm. The fallthrough `else` - // branch, when present, contains `stab_clause`s that count for - // standard. The `with` Call itself is a multi-arm container Call - // that contributes once to modified CCN. + // nodes — NOT `stab_clause`s — and each is a decision in both tiers + // (#1535). The fallthrough `else` branch, when present, contains + // `stab_clause`s that count for standard, and collapses into one + // modified decision. #[cfg(feature = "elixir")] #[test] - fn elixir_with_else_only_counts_else_arms() { + fn elixir_with_counts_arrow_clauses_and_else_arms() { check_metrics::( "defmodule Foo do\n def f(x) do\n with {:ok, v} <- fetch(x),\n {:ok, w} <- fetch(v) do\n {:ok, w}\n else\n :error -> :nope\n other -> {:bad, other}\n end\n end\nend\n", "foo.ex", |metric| { - // standard: 3 entries + 2 else-block stabs = 5 - // modified: 3 entries + 1 with Call = 4 - assert_eq!(metric.cyclomatic.cyclomatic_sum(), 5); - assert_eq!(metric.cyclomatic.cyclomatic_modified_sum(), 4); + // standard: 3 entries + 2 `<-` clauses + 2 else-block + // stabs = 7 (#1535 added the `<-` clauses) + // modified: 3 entries + 2 `<-` clauses + 1 else + // dispatch = 6 + assert_eq!(metric.cyclomatic.cyclomatic_sum(), 7); + assert_eq!(metric.cyclomatic.cyclomatic_modified_sum(), 6); + }, + ); + } + + // A `with`'s `<-` clauses and a `for … reduce:`'s head clause + // (#1535). Each row is one `def`, read as (cyclomatic, modified, + // abc.conditions); the function's base path is the 1. + // + // Expected values, from the twins rather than the fix (measured, + // `--no-config`, as cyclomatic / modified / ABC conditions): + // - `w1`: one `<-` is one decision, Rust's + // `if let Ok(a) = g(x) { a }` — 2/2/1, and `g(x)?` is 2/2/0. + // - `w2` / `w2d`: two `<-` are two, Rust's nested `if let` — 3/3/2. + // `w2d`'s `else _ -> 0` picks between nothing, so it is free in + // both tiers, as Rust's `else { 0 }` is. + // - `w1e`: the `else {:error, e} ->` arm pays per clause as a + // `case` arm does, and its dispatch collapses into one modified + // decision: 1 + `<-` + arm = 3, and 3 modified. `w1n`'s named + // `e ->` binds, so it is no bare `_` and scores the same. `w1g`'s + // guarded `_ when x > 1 ->` is a decision with a guard on it: + // 1 + `<-` + clause + guard = 4, and modified collapses only the + // clause — 4. + // - `w2e`: 1 + two `<-` + two arms = 5; modified 1 + 2 + 1 = 4. + // - `wg`: the guard is one more decision on the `<-`, Rust's + // `if let Ok(a) = g(x) && a > 0` — 3/3/2. + // - `wb`: a bare match `b = …` in a `with` cannot fail over to the + // `else`, so it is no clause — 2/2/1, `w1`'s value. + // - `wk` / `wke`: the keyword form scores as the block form does. + // - `r1` / `rk1`: a single reduce clause always matches, so the + // reduce scores what `fp`, the `for` without one, does — 2/2/0. + // - `r2` / `r2d`: two reduce clauses are one decision, n−1 as a + // multi-clause `fn` is (#776), the trailing `_ ->` included — + // 3/2/1. `rk2` is `r2` in the keyword form. + // - `fp` / `ff` / `fg` / `fb`: a `for` generator's `<-` is not a + // `with` clause, nor is a guard on it (`fg` scores as the bare + // filter `ff`), nor a bitstring generator's — each scores the + // `for`'s one decision, as before #1535. + // - `c`: `case {:ok, a} / _`, the Rust `match` twin — 2/2/1. + // + // ABC conditions sit at `cyclomatic - 1` on every row but the + // `for`s, whose loop decision ABC deliberately does not count. + #[cfg(feature = "elixir")] + #[test] + fn elixir_with_arrow_clauses_and_reduce_head_1535() { + let src = "defmodule M do + def w1(x) do + with {:ok, a} <- g(x) do a end + end + def w2(x) do + with {:ok, a} <- g(x), {:ok, b} <- h(a) do b end + end + def w2d(x) do + with {:ok, a} <- g(x), {:ok, b} <- h(a) do b else _ -> 0 end + end + def w1e(x) do + with {:ok, a} <- g(x) do a else {:error, e} -> e end + end + def w1n(x) do + with {:ok, a} <- g(x) do a else e -> e end + end + def w1g(x) do + with {:ok, a} <- g(x) do a else _ when x > 1 -> 0 end + end + def w2e(x) do + with {:ok, a} <- g(x), {:ok, b} <- h(a) do + b + else + {:error, e} -> e + :nope -> 0 + end + end + def wg(x) do + with {:ok, a} when is_integer(a) <- g(x) do a end + end + def wb(x) do + with {:ok, a} <- g(x), b = a + 1 do b end + end + def wk(x), do: with({:ok, a} <- g(x), do: a, else: (_ -> 0)) + def wke(x), do: with({:ok, a} <- g(x), do: a, else: ({:error, e} -> e)) + def r1(xs) do + for x <- xs, reduce: 0 do + acc -> acc + x + end + end + def rk1(xs), do: for(x <- xs, reduce: 0, do: (acc -> acc + x)) + def r2(xs) do + for x <- xs, reduce: 0 do + 0 -> x + acc -> acc + x + end + end + def r2d(xs) do + for x <- xs, reduce: 0 do + 0 -> x + _ -> 1 + end + end + def rk2(xs), do: for(x <- xs, reduce: 0, do: (0 -> x; acc -> acc + x)) + def fp(xs), do: for(x <- xs, do: x) + def ff(xs), do: for(x <- xs, x > 1, do: x) + def fg(xs), do: for(x when x > 1 <- xs, do: x) + def fb(s), do: for(<>, do: c) + def c(x) do + case g(x) do + {:ok, a} -> a + _ -> 0 + end + end +end +"; + let expected: &[(&str, u64, u64, u64)] = &[ + ("w1", 2, 2, 1), + ("w2", 3, 3, 2), + ("w2d", 3, 3, 2), + ("w1e", 3, 3, 2), + ("w1n", 3, 3, 2), + ("w1g", 4, 4, 3), + ("w2e", 5, 4, 4), + ("wg", 3, 3, 2), + ("wb", 2, 2, 1), + ("wk", 2, 2, 1), + ("wke", 3, 3, 2), + ("r1", 2, 2, 0), + ("rk1", 2, 2, 0), + ("r2", 3, 2, 1), + ("r2d", 3, 2, 1), + ("rk2", 3, 2, 1), + ("fp", 2, 2, 0), + ("ff", 2, 2, 1), + ("fg", 2, 2, 1), + ("fb", 2, 2, 0), + ("c", 2, 2, 1), + ]; + check_func_space_only::( + src, + "foo.ex", + &[Metric::Cyclomatic, Metric::Abc], + |space| { + let module = &space.spaces[0]; + assert_eq!(module.spaces.len(), expected.len(), "fixture members"); + for &(name, cyclomatic, modified, conditions) in expected { + let f = child_space(module, name); + let measured = ( + f.metrics.cyclomatic.cyclomatic(), + f.metrics.cyclomatic.cyclomatic_modified(), + f.metrics.abc.conditions(), + ); + assert_eq!(measured, (cyclomatic, modified, conditions), "{name}"); + } }, ); } diff --git a/src/metrics/cyclomatic/elixir.rs b/src/metrics/cyclomatic/elixir.rs index 40108a9f..88b62aa9 100644 --- a/src/metrics/cyclomatic/elixir.rs +++ b/src/metrics/cyclomatic/elixir.rs @@ -19,22 +19,25 @@ impl Cyclomatic for ElixirCode { // The split between standard and modified CCN mirrors the C-family // case/switch treatment: per-arm `stab_clause` nodes contribute // standard, while the multi-arm container Calls (`case`/`cond`/ - // `with`/`try`) contribute modified. Single-branch keyword Calls - // (`if`/`unless`/`for`/`while`) contribute to both. Short-circuit - // booleans (`&&`, `||`, `and`, `or`) contribute to both. + // `try`) contribute modified. Single-branch keyword Calls + // (`if`/`unless`/`for`/`while`) contribute to both, as do a `with`'s + // `<-` clauses and short-circuit booleans (`&&`, `||`, `and`, `or`). fn compute<'a>( node: &Node<'a>, code: &'a [u8], ancestors: Ancestors<'a, '_>, stats: &mut Stats, ) { + // bca: suppress(cyclomatic, halstead) — kind dispatch table; each gated arm is one rule use Elixir as E; match node.kind_id().into() { // Per-arm decisions: each `stab_clause` is one arm of a // `case`/`cond`/`with`/anonymous-fn body or a `rescue`/ - // `catch` handler. Standard-only — modified counts the - // container Call once. + // `catch` handler. Standard-only: modified counts a + // `case`/`cond`/`try` container Call once, a `with` once + // only when its `else` dispatches (the `"with"` arm below), + // and gives a multi-clause `fn` or `reduce` nothing. // // The exception is the *first* `stab_clause` of an // `anonymous_function` (`fn … -> … end`): it is the @@ -44,9 +47,9 @@ impl Cyclomatic for ElixirCode { // `getter::elixir` → `SpaceKind::Function`), so counting the // head clause too over-reports a trivial `fn x -> x end` as // 2 (issue #776). Only the 2nd+ clauses of a multi-clause - // `fn` are real branches. `case`/`cond`/`with` arms have a - // `do_block` parent — not `anonymous_function` — so they are - // unaffected and keep counting. + // `fn` are real branches. A `for … reduce:`'s accumulator + // clauses take the same skip (#1535): a single `acc -> …` + // always matches. `case`/`cond`/`with` arms keep counting. // // Two further clause shapes are the construct's default arm // and are excluded to match the sibling family — Rust's @@ -99,6 +102,19 @@ impl Cyclomatic for ElixirCode { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } + // A `with`'s `pattern <- expr` clause either matches or + // short-circuits to the `else`, so each is a decision + // (#1535) — the Rust `if let` / `?` twin pays one in both + // tiers. The clauses run in sequence rather than being + // alternatives of one dispatch, so modified does not + // collapse them. A `for` generator's `<-` is excluded by + // the gate: the `for` already pays its loop decision. + E::LTDASH + if crate::metrics::npa::elixir_arrow_is_with_clause(node, code, ancestors) => + { + stats.cyclomatic += 1.; + stats.cyclomatic_modified += 1.; + } // Short-circuit booleans add a decision point in both // metrics — where applied. `&and/2`, `&||/2` and // `Kernel.||(a, b)` only name the operator and decide @@ -128,7 +144,15 @@ impl Cyclomatic for ElixirCode { // (the container collapses to a single decision). // Per-arm `stab_clause`s already contribute to // standard above. - "case" | "cond" | "with" | "try" => { + "case" | "cond" | "try" => { + stats.cyclomatic_modified += 1.; + } + // A `with`'s decisions are its `<-` clauses, + // counted above in both tiers. What modified + // collapses is its `else`, the dispatch over + // the failed value — once, and only when that + // dispatch picks between anything (#1535). + "with" if crate::metrics::npa::elixir_with_else_dispatches(node, code) => { stats.cyclomatic_modified += 1.; } _ => {} diff --git a/src/metrics/npa/shared.rs b/src/metrics/npa/shared.rs index 3a46064f..60888332 100644 --- a/src/metrics/npa/shared.rs +++ b/src/metrics/npa/shared.rs @@ -668,19 +668,70 @@ pub(crate) fn ruby_in_clause_counts(in_clause: &Node, source: &[u8]) -> bool { }) } -/// Whether `node` is a `when` `binary_operator` — the shape a repeated -/// guard's alternatives nest through. +/// Whether `node` is a `binary_operator` applying `operator`. /// /// `binary_operator` carries three kind aliases at this pin so it is /// matched by rule name rather than by enumerating ids -/// (grammar-dispatch §1); `when` has exactly one id. -fn elixir_is_when_operator(node: &Node) -> bool { +/// (grammar-dispatch §1); `when` and `<-` each have exactly one id. +fn elixir_is_binary_operator(node: &Node, operator: Elixir) -> bool { const BINARY_OPERATOR: &str = "binary_operator"; node.kind() == BINARY_OPERATOR && node .child_by_field_name("operator") - .is_some_and(|operator| operator.kind_id() == Elixir::When as u16) + .is_some_and(|token| token.kind_id() == operator as u16) +} + +/// Whether `node` is a `when` `binary_operator` — the shape a repeated +/// guard's alternatives nest through. +fn elixir_is_when_operator(node: &Node) -> bool { + elixir_is_binary_operator(node, Elixir::When) +} + +/// Whether the two ancestors `up` yields first are a `with` Call's +/// `arguments` and the Call itself — the position of a `with` clause. +/// +/// `arguments` carries five kind aliases at this pin, so it is matched +/// by rule name (grammar-dispatch §1). +fn elixir_is_with_argument<'a>(mut up: impl Iterator>, code: &'a [u8]) -> bool { + up.next() + .is_some_and(|arguments| arguments.kind() == "arguments") + && up.next().is_some_and(|call| { + crate::lang_helpers::elixir::elixir_call_keyword(&call, code) == Some("with") + }) +} + +/// Whether `node`, a `<-` token, is the operator of one of a `with`'s +/// `pattern <- expr` clauses: a decision, since the value either +/// matches the pattern or short-circuits to the `else` clauses (#1535). +/// It is the shape a Rust `if let` or `?` has, both of which pay one +/// decision in standard and modified cyclomatic alike. +/// +/// Only a direct argument of a `with` qualifies. The same token is a +/// `for` comprehension's generator (`for x <- xs`), which filters +/// rather than short-circuits and is deliberately left alone — `for` +/// already pays one decision for its loop — and a bitstring +/// generator (`for <>`) sits inside a `bitstring`, not under +/// `arguments`. +/// +/// Position alone is the gate, so it needs no separate test that the +/// token is applied (#1534): in every shape probed where it is not — +/// named (`with &<-/2`) or stranded by error recovery (`with(<-)`, +/// `with a, <- b`) — it sits under an `operator_identifier`, whose +/// parent is never the `with`'s `arguments`. +/// +/// A clause wrapped in parentheses after a space (`with (a <- b) do`) +/// parses through a `block` and is not recognised; Elixir drops the +/// parentheses, so it is a clause, but the spelling is rare. +/// +/// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode` +/// (grammar-dispatch §7). +pub(crate) fn elixir_arrow_is_with_clause<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> bool { + elixir_is_with_argument(ancestors.iter(node).skip(1).map(|(a, _)| a), code) } /// The one alternative a guard's `when` token introduces. @@ -718,8 +769,9 @@ pub(crate) fn elixir_when_alternative<'a>(when_operator: &Node<'a>) -> Option( code: &'a [u8], ancestors: Ancestors<'a, '_>, ) -> bool { - use Elixir as E; - - const ARGUMENTS: &str = "arguments"; - let mut chain = ancestors.iter(node); // The token's parent is the `when` operator node itself; the first // ancestor above the chain of `when` operators is the position that @@ -772,13 +820,45 @@ pub(crate) fn elixir_when_is_guard<'a>( let Some((parent, _)) = above else { return false; }; + elixir_is_guard_position( + &operator, + &parent, + chain.map(|(ancestor, _)| ancestor), + code, + ) +} + +/// Whether `parent` — the first ancestor above the chain of `when` +/// operators that `operator` tops — holds that chain in a guard +/// position; `up` yields the ancestors above `parent`. The second step +/// of [`elixir_when_is_guard`], after the climb. +fn elixir_is_guard_position<'a>( + operator: &Node<'a>, + parent: &Node<'a>, + mut up: impl Iterator>, + code: &'a [u8], +) -> bool { + use Elixir as E; + + const ARGUMENTS: &str = "arguments"; + if parent.kind_id() == E::StabClause as u16 { return parent .child_by_field_name("left") .is_some_and(|left| left.id() == operator.id()); } + // A guard on the pattern of a `with` clause. `when` binds tighter + // than `<-`, so it parses as the clause's `left`; a `when` on the + // right would be a guard outside any guard position, which Elixir + // rejects, so the side is not checked. The guard of a `for` + // generator (`for x when x > 1 <- xs`) is a filter, which the + // comprehension's own decision already covers exactly as a bare + // filter (`for x <- xs, x > 1`) is — so it is not a guard here. + if elixir_is_binary_operator(parent, E::LTDASH) { + return elixir_is_with_argument(up, code); + } parent.kind() == ARGUMENTS - && chain.next().is_some_and(|(call, _)| { + && up.next().is_some_and(|call| { crate::lang_helpers::elixir::elixir_call_keyword(&call, code).is_some_and(|keyword| { crate::lang_helpers::elixir::elixir_is_method_macro(keyword) || matches!(keyword, "defguard" | "defguardp") @@ -812,9 +892,16 @@ pub(crate) fn elixir_sole_unguarded_pattern<'a>(node: &Node<'a>) -> Option bool { + pattern.kind_id() == Elixir::Identifier as u16 && pattern.utf8_text(code) == Some("_") +} + /// Returns `true` when `node` is a construct's free default clause: -/// a bare `_ ->` catch-all outside an anonymous fn, or an unguarded -/// `true ->` directly under a `cond`. +/// a bare `_ ->` catch-all outside a head-skipping construct (an +/// anonymous fn or a `for … reduce:`), or an unguarded `true ->` +/// directly under a `cond`. /// /// Both shapes hinge on the clause's sole unguarded pattern, and /// [`elixir_sole_unguarded_pattern`]'s child scan allocates a cursor @@ -829,12 +916,13 @@ pub(crate) fn elixir_sole_unguarded_pattern<'a>(node: &Node<'a>) -> Option` too would -/// leave `fn 0 -> :a; _ -> :b end` at zero decisions while the -/// identical `case` reports one. +/// NOT apply when `skips_head` — the clause's container is an +/// `anonymous_function` or a `for … reduce:`: such a multi-clause +/// dispatch is like `case` — n clauses are n−1 decisions — and its free +/// base path is already granted by the head-clause skip (#776, #1535), +/// so excluding a trailing `_ ->` too would leave +/// `fn 0 -> :a; _ -> :b end` at zero decisions while the identical +/// `case` reports one. /// /// `true ->` under `cond`: the exclusion is *shape-based* — any /// unguarded `true ->` whose parent is the `do_block` of a `Call` @@ -849,6 +937,7 @@ fn elixir_is_default_clause<'a>( node: &Node<'a>, code: &'a [u8], ancestors: Ancestors<'a, '_>, + skips_head: bool, ) -> bool { use Elixir as E; @@ -856,44 +945,56 @@ fn elixir_is_default_clause<'a>( return false; }; match pattern.kind_id().into() { - // Bare `_` catch-all — free everywhere except under an - // anonymous fn, whose free base path the head-clause skip - // already provides. - E::Identifier => { - pattern.utf8_text(code) == Some("_") - && ancestors - .parent(node) - .is_none_or(|parent| parent.kind_id() != E::AnonymousFunction as u16) - } + // Bare `_` catch-all — free everywhere except where the + // head-clause skip already provides the free base path. + E::Identifier => !skips_head && elixir_is_bare_wildcard(&pattern, code), // Unguarded `true` — free only as `cond`'s designated // default; the O(1) parent/grandparent checks run after the // text compare so non-`true` booleans bail early. E::Boolean => { - pattern.utf8_text(code) == Some("true") && elixir_is_cond_clause(node, code, ancestors) + pattern.utf8_text(code) == Some("true") + && elixir_do_section_call(node, code, ancestors).is_some_and(|call| { + crate::lang_helpers::elixir::elixir_call_keyword(&call, code) == Some("cond") + }) } _ => false, } } -/// Returns `true` when `node` is the first `stab_clause` child of an -/// `anonymous_function` parent — the closure's head clause rather than a -/// pattern-dispatch branch. +/// Whether `node`'s clauses-owner grants a free head clause: an +/// `anonymous_function` (#776), or a `for … reduce:` comprehension +/// (#1535), whose accumulator clauses are an anonymous fn in all but +/// spelling — a single `acc -> …` always matches, and only the 2nd+ +/// clauses dispatch. /// -/// The grammar shape is `anonymous_function → fn stab_clause+ end`, so -/// the first child whose kind is `stab_clause` (skipping the `fn` token) -/// is the head. Multi-clause `fn`s thus skip only their first clause; -/// every other construct's clauses have a block parent and never match -/// here (issue #776). -fn elixir_is_anonymous_fn_head_clause<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - use Elixir as E; +/// A `for` holds clauses in its `do` section only under `reduce:`; +/// without it the section is a plain body. So the Call's keyword alone +/// identifies the shape, with no need to find the `reduce:` pair. +fn elixir_clauses_skip_head<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> bool { + ancestors.parent_has_kind(node, Elixir::AnonymousFunction as u16) + || elixir_do_section_call(node, code, ancestors).is_some_and(|call| { + crate::lang_helpers::elixir::elixir_call_keyword(&call, code) == Some("for") + }) +} +/// Returns `true` when `node` is its parent's first `stab_clause` — +/// the head clause of a construct [`elixir_clauses_skip_head`] names. +/// +/// The parent is the `anonymous_function` (`fn stab_clause+ end`), the +/// `do_block` (`do stab_clause+ end`) or the keyword form's +/// parenthesised `block`; in each the first child whose kind is +/// `stab_clause` (skipping the `fn` / `do` / `(` token) is the head. +fn elixir_is_first_clause<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { ancestors .parent(node) - .filter(|parent| parent.kind_id() == E::AnonymousFunction as u16) .and_then(|parent| { parent .children() - .find(|child| child.kind_id() == E::StabClause as u16) + .find(|child| child.kind_id() == Elixir::StabClause as u16) }) .is_some_and(|first| first.id() == node.id()) } @@ -907,7 +1008,10 @@ fn elixir_is_anonymous_fn_head_clause<'a>(node: &Node<'a>, ancestors: Ancestors< /// An anonymous fn's head clause is the closure's definition, not a /// dispatch: the closure opens its own function space whose base path /// already covers it, so counting it too would score a trivial -/// `fn x -> x end` as a decision (#776). +/// `fn x -> x end` as a decision (#776). A `for … reduce:`'s first +/// accumulator clause is skipped for the same reason: the `for` already +/// pays its one decision, and a single `acc -> …` clause always matches +/// (#1535). /// /// The parent is allowlisted because the grammar puts a `stab_clause` /// in one more position: a parenthesised `block`, which is how a @@ -944,9 +1048,57 @@ pub(crate) fn elixir_clause_is_decision<'a>( .is_some_and(|pair| elixir_is_section_pair(&pair, code, SECTION_KEYS)), _ => false, }); - in_section - && !elixir_is_anonymous_fn_head_clause(node, ancestors) - && !elixir_is_default_clause(node, code, ancestors) + if !in_section { + return false; + } + let skips_head = elixir_clauses_skip_head(node, code, ancestors); + if skips_head && elixir_is_first_clause(node, ancestors) { + return false; + } + !elixir_is_default_clause(node, code, ancestors, skips_head) +} + +/// Whether a `with` Call's `else` clauses dispatch: whether at least one +/// of them is a decision rather than the free bare `_ ->` default. +/// +/// Modified cyclomatic collapses a clause construct's arms into one +/// decision, which for `case` / `cond` / `try` is the container paying +/// once. A `with`'s decisions are its `<-` clauses, each paid on its own +/// in both tiers as a Rust `if let` is; its `else` is the dispatch over +/// the failed value, which collapses into one decision exactly when it +/// holds one — so `else _ -> 0`, which picks nothing, stays free in +/// modified as it is in standard, like Rust's `if let … else` (#1535). +/// Spelled in either form: the block form's `else_block`, or the +/// keyword form's `else:` pair whose value is a parenthesised `block`. +/// +/// In a `with`'s `else` the default-clause rule reduces to the bare +/// `_`: `true ->` is free only under a `cond`, and the head skip only +/// under a `fn` or `reduce:`, so [`elixir_clause_is_decision`] agrees +/// clause by clause with the test below. +pub(crate) fn elixir_with_else_dispatches(call: &Node, code: &[u8]) -> bool { + use Elixir as E; + + call.children().any(|child| { + let section = match child.kind_id().into() { + E::DoBlock => child + .children() + .find(|c| c.kind_id() == E::ElseBlock as u16), + _ if child.kind() == "arguments" => child + .children() + .filter(|c| c.kind_id() == E::Keywords as u16) + .flat_map(|keywords| keywords.children()) + .find(|pair| elixir_is_section_pair(pair, code, &["else:"])) + .and_then(|pair| pair.child_by_field_name("value")), + _ => None, + }; + section.is_some_and(|section| { + section.children().any(|clause| { + clause.kind_id() == E::StabClause as u16 + && !elixir_sole_unguarded_pattern(&clause) + .is_some_and(|pattern| elixir_is_bare_wildcard(&pattern, code)) + }) + }) + }) } /// The keyword spellings of the clause sections a `do … end` block @@ -965,29 +1117,26 @@ fn elixir_is_section_pair(pair: &Node, code: &[u8], keys: &[&str]) -> bool { .is_some_and(|key| keys.contains(&key.trim_end())) } -/// Whether `node` (a `stab_clause`) is one arm of a `cond`: it sits in -/// the `do` section of a `Call` spelling `cond`, either the block form's -/// `do_block` (whose parent is the call) or the keyword form's -/// `do: (…)` block (`block` → `pair` → `keywords` → `arguments` → call). -fn elixir_is_cond_clause<'a>( +/// The `Call` whose `do` section holds `node` (a `stab_clause`): the +/// block form's `do_block` (whose parent is the call) or the keyword +/// form's `do: (…)` block (`block` → `pair` → `keywords` → `arguments` +/// → call). The caller asks the Call's keyword (`cond`, `for`). +fn elixir_do_section_call<'a>( node: &Node<'a>, code: &'a [u8], ancestors: Ancestors<'a, '_>, -) -> bool { +) -> Option> { use Elixir as E; let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); - let call = match up.next().map(|parent| parent.kind_id().into()) { + match up.next().map(|parent| parent.kind_id().into()) { Some(E::DoBlock) => up.next(), Some(E::Block) => up .next() .filter(|pair| elixir_is_section_pair(pair, code, &["do:"])) .and_then(|_| up.nth(2)), _ => None, - }; - call.is_some_and(|call| { - crate::lang_helpers::elixir::elixir_call_keyword(&call, code) == Some("cond") - }) + } } // A `visibility_modifier` node counts as public unless it has a direct From ca9ec653b0e09c93771f3f307cec76c4ea5668a7 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:52:59 -0700 Subject: [PATCH 13/48] fix(getter/csharp): bill a lambda's static/async tree-sitter-c-sharp aliases a lambda's or anonymous method's bare `static` / `async` onto a childless `modifier` node, which no Halstead half billed, while the declaration spelling of both keywords is an operator through its leaf. Bill both texts as operators in the aliased-modifier arm, and list `Static` / `Async` in `CsharpCode::is_primitive` so the two spellings share the lexeme-keyed map instead of splitting one keyword across both operator maps and counting it twice in `n1`. The comment there claimed widening the getter needed no change to `is_primitive`; it did. Adds a `parameters.cs` corpus fixture carrying every aliased modifier spelling, which also gives #1418 its first snapshot coverage. Fixes #1482 --- big-code-analysis-ast/src/checker.rs | 7 +- big-code-analysis-ast/src/checker/csharp.rs | 46 ++++--- big-code-analysis-ast/src/getter/csharp.rs | 25 ++-- big-code-analysis-book/src/metrics.md | 14 ++- src/metrics/halstead.rs | 133 ++++++++++++++------ tests/corpus/csharp_test.rs | 2 +- tests/repositories/big-code-analysis-output | 2 +- 7 files changed, 144 insertions(+), 85 deletions(-) diff --git a/big-code-analysis-ast/src/checker.rs b/big-code-analysis-ast/src/checker.rs index 88337697..212f0d78 100644 --- a/big-code-analysis-ast/src/checker.rs +++ b/big-code-analysis-ast/src/checker.rs @@ -829,11 +829,8 @@ pub fn csharp_member_has_accessors(node: &Node) -> bool { /// - `_parameter_type_with_modifiers` — `this`, `scoped`, `ref`, `out`, /// `in`, `readonly` in parameter position, the six #1418 bills. /// - `_lambda_expression_init` and `anonymous_method_expression` — -/// `static` and `async` on a lambda or `delegate { … }`. These reach -/// [`CsharpCode::get_op_type_with_code`]'s fallthrough and stay -/// unclassified, exactly as they were before #1418; billing them is a -/// separate decision, since the *declaration* spelling of the same two -/// keywords is billed through its leaf. +/// `static` and `async` on a lambda or `delegate { … }`, billed as +/// operators like their declaration spelling (#1482). /// /// Child-presence is therefore what separates an alias from the rule, /// and it is the grammar's shape rather than an inference about a broken diff --git a/big-code-analysis-ast/src/checker/csharp.rs b/big-code-analysis-ast/src/checker/csharp.rs index 23c385ff..de540d54 100644 --- a/big-code-analysis-ast/src/checker/csharp.rs +++ b/big-code-analysis-ast/src/checker/csharp.rs @@ -118,32 +118,28 @@ impl Checker for CsharpCode { // them through the lexeme-keyed `primitive_operators` map so // distinct keywords count as distinct operators (issue #286). // - // The five parameter-modifier keywords join it for the mirror - // reason (#1418). Each has *two* kind spellings for one - // operator: its own token, and the childless `modifier` the - // grammar aliases it to in parameter position. Keying only the - // alias by text would split one `ref` across the two maps and - // count it twice in `n1` — the #453 shape — while keying the - // alias by `kind_id` would instead collapse all five spellings - // into one operator that `bca ops` renders as the word - // `modifier`, which appears nowhere in the source. Text is the - // only key under which the two positions are the same operator. + // Every keyword the grammar aliases to a childless `modifier` + // joins it for the mirror reason: the five parameter modifiers + // (#1418) and a lambda's `static` / `async` (#1482). Each has + // *two* kind spellings for one operator: its own token, and the + // aliased `modifier`. Keying only the alias by text would split + // one `ref` or `static` across the two maps and count it twice + // in `n1` — the #453 shape — while keying the alias by `kind_id` + // would instead collapse every spelling into one operator that + // `bca ops` renders as the word `modifier`, which appears + // nowhere in the source. Text is the only key under which the + // two positions are the same operator, so the bare kind of every + // keyword the getter bills through the alias must be listed too. // - // Listing the five bare kinds moves no existing count: each - // renders exactly one text through `get_operator_id_as_str` and - // no other kind renders that text, so the vocabulary entry only - // changes which map holds it. + // Listing the bare kinds moves no existing count: each renders + // exactly one text through `get_operator_id_as_str` and no other + // kind renders that text, so the vocabulary entry only changes + // which map holds it. // - // The shared predicate is wider than those five, and deliberately - // so: it holds for every aliased `modifier`, including the `this` - // receiver that `get_op_type_with_code` bills as an operand and - // the `static` / `async` of a lambda that it leaves unclassified. - // Neither is a disagreement between the two methods, because - // `compute_halstead` consults this one only under its `Operator` - // arm — so the answer is never read for anything the getter did - // not call an operator. Widening the getter later (billing a - // lambda's `static`) therefore needs no change here, and gets the - // lexeme key it would want. + // The shared predicate also holds for the aliased `this` + // receiver, which `get_op_type_with_code` bills as an operand. + // That is no disagreement, because `compute_halstead` consults + // this method only under its `Operator` arm. matches!( node.kind_id().into(), Csharp::PredefinedType @@ -152,6 +148,8 @@ impl Checker for CsharpCode { | Csharp::In | Csharp::Readonly | Csharp::Scoped + | Csharp::Static + | Csharp::Async ) || csharp_is_aliased_modifier(node) } } diff --git a/big-code-analysis-ast/src/getter/csharp.rs b/big-code-analysis-ast/src/getter/csharp.rs index fa97fb89..c3fa305f 100644 --- a/big-code-analysis-ast/src/getter/csharp.rs +++ b/big-code-analysis-ast/src/getter/csharp.rs @@ -193,7 +193,7 @@ impl Getter for CsharpCode { // (grammar-dispatch section 7). // // `CsharpCode::is_primitive` routes the operator half through the - // lexeme-keyed map, because these five keywords now have two kind + // lexeme-keyed map, because these seven keywords have two kind // spellings for one operator; its comment carries that argument. fn get_op_type_with_code<'a>( node: &Node<'a>, @@ -205,18 +205,17 @@ impl Getter for CsharpCode { } match node_text(code, node) { Some("this") => TokenRole::Operand, - Some("ref" | "out" | "in" | "scoped" | "readonly") => TokenRole::Operator, - // Live, not merely defensive: `_lambda_expression_init` and - // `anonymous_method_expression` alias a bare `static` / - // `async` onto `modifier` too, so `static (int x) => x` - // arrives here. Leaving those unclassified is what they - // already were before #1418, and closing that gap is a - // decision about lambdas rather than about parameters — the - // *declaration* spelling of both keywords is billed through - // its leaf, so they are unbilled only in this one position. - // A grammar bump that adds a seventh parameter spelling - // lands here as well, unclassified rather than billed as - // whichever arm happened to be last. + // `_lambda_expression_init` and `anonymous_method_expression` + // alias a bare `static` / `async` onto `modifier` too, so + // `static (int x) => x` arrives here. The declaration spelling + // of both keywords is billed as an operator through its leaf, + // so the lambda spelling is billed the same way (#1482). + Some("ref" | "out" | "in" | "scoped" | "readonly" | "static" | "async") => { + TokenRole::Operator + } + // A grammar bump that adds another aliased spelling lands + // here unclassified rather than billed as whichever arm + // happened to be last. _ => TokenRole::Unknown, } } diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 8ea1307c..efb70f0e 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -703,12 +703,14 @@ tokens involved the same way they spell real operators: `? super T` in Java and Groovy (#1380). - **A keyword's role follows its text, not its node kind.** C#'s grammar parses the bare `this`, `ref`, `out`, `in`, `scoped` and - `readonly` of a parameter as a `modifier` node, a different kind - from the one the same keyword has anywhere else. Each keeps the - role its text has elsewhere: an extension method's `this Foo f` is - an operand, and the `ref` in `M(ref int a)` is the same operator as - the `ref` in the call `M(ref x)`, adding to `total_operators` but - not to `unique_operators` (#1418). + `readonly` of a parameter, and the `static` / `async` of a lambda, + as a `modifier` node, a different kind from the one the same + keyword has anywhere else. Each keeps the role its text has + elsewhere: an extension method's `this Foo f` is an operand, the + `ref` in `M(ref int a)` is the same operator as the `ref` in the + call `M(ref x)`, and `static (int x) => x` bills the same `static` + as `static void M()`. A repeated operator keyword adds to + `total_operators` but not to `unique_operators` (#1418, #1482). - **A named operator is one operand.** Elixir names an operator without applying it in a capture (`&==/2`, `&and/2`) or a remote call (`Kernel.||(a, b)`), and its grammar wraps the token in an diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index e2c2c121..6732aef8 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -3678,48 +3678,63 @@ mod tests { // assertion is dead. That left four of the five bare kinds with // no input at all — dropping `Csharp::Ref`, `In`, `Readonly` or // `Scoped` from `is_primitive` failed no test in the workspace. - let both_code = CSHARP_MODIFIER_BOTH_SPELLINGS.as_bytes(); - let spellings = [ - ("ref", Csharp::Ref as u16), - ("out", Csharp::Out as u16), - ("in", Csharp::In as u16), - ("scoped", Csharp::Scoped as u16), - ("readonly", Csharp::Readonly as u16), - ]; - let (mut bare, mut aliased) = ([0_usize; 5], [0_usize; 5]); + assert_modifier_spellings_are_one_operator( + CSHARP_MODIFIER_BOTH_SPELLINGS, + &[ + ("ref", Csharp::Ref as u16), + ("out", Csharp::Out as u16), + ("in", Csharp::In as u16), + ("scoped", Csharp::Scoped as u16), + ("readonly", Csharp::Readonly as u16), + ], + ); + } + + /// Asserts each keyword in `spellings` appears in `source` both as + /// its own `kind` and as a childless aliased `modifier`, and that the + /// two spellings render as exactly one entry in the operator + /// vocabulary. + /// + /// The vocabulary is the concatenation of the kind-keyed and the + /// lexeme-keyed operator maps, so a keyword listed twice is one + /// keyword split across both maps and counted twice in `n1`. + #[cfg(feature = "csharp")] + #[track_caller] + fn assert_modifier_spellings_are_one_operator(source: &str, spellings: &[(&str, u16)]) { + let code = source.as_bytes(); + // (bare, aliased) occurrences per keyword. + let mut seen = vec![(0_usize, 0_usize); spellings.len()]; - for_each_node_with_chain::(both_code, |node, _| { - let text = &both_code[node.start_byte()..node.end_byte()]; + for_each_node_with_chain::(code, |node, _| { + let text = &code[node.start_byte()..node.end_byte()]; let Some(index) = spellings.iter().position(|(kw, _)| kw.as_bytes() == text) else { return; }; if node.kind_id() == Csharp::Modifier as u16 && node.child_count() == 0 { - aliased[index] += 1; + seen[index].1 += 1; } else if node.kind_id() == spellings[index].1 { - bare[index] += 1; + seen[index].0 += 1; } }); - // Without this the count below is vacuous for any keyword the - // fixture stopped spelling both ways — the same decay that hid - // the four unguarded kinds in the first place. - for (index, (keyword, _)) in spellings.iter().enumerate() { + // Without this the vocabulary count below is vacuous for any + // keyword the fixture stopped spelling both ways — the decay + // that once left four of #1418's bare kinds with no input. + for ((keyword, _), (bare, aliased)) in spellings.iter().zip(&seen) { assert!( - bare[index] > 0 && aliased[index] > 0, - "fixture must spell `{keyword}` both ways; it has {} bare and {} aliased", - bare[index], - aliased[index] + *bare > 0 && *aliased > 0, + "fixture must spell `{keyword}` both ways; it has {bare} bare and {aliased} aliased" ); } - let both = ops_of::(CSHARP_MODIFIER_BOTH_SPELLINGS, "both.cs"); + let ops = ops_of::(source, "both.cs"); for (keyword, _) in spellings { assert_eq!( - both.operators.iter().filter(|o| *o == keyword).count(), + ops.operators.iter().filter(|o| o == keyword).count(), 1, "`{keyword}` must be one operator across both of its kind spellings; \ operators were {:?}", - both.operators + ops.operators ); } } @@ -3738,11 +3753,8 @@ mod tests { /// Childlessness says "aliased", *not* "parameter modifier" — /// `_lambda_expression_init` and `anonymous_method_expression` alias /// a bare `static` / `async` onto the same kind, which the second - /// walk below pins so the distinction cannot be forgotten. Their - /// `Unknown` is today's behaviour rather than a contract: it is what - /// they were before #1418, and billing them is a decision about - /// lambdas. It is asserted only so making that decision has to come - /// through this test. + /// walk below pins so the distinction cannot be forgotten. Both are + /// operators, as their declaration spelling is (#1482). #[cfg(feature = "csharp")] #[test] fn csharp_aliased_modifier_is_childless() { @@ -3759,9 +3771,8 @@ mod tests { if node.child_count() == 0 { aliased += 1; // Spelled out rather than `this`-versus-everything-else, - // because "everything else is an operator" is false of - // the kind at large — a lambda's aliased `static` is - // neither, and the second walk below owns it. + // so a text this fixture does not carry fails loudly; + // the lambda spellings are owned by the second walk. let (correct, expected) = match text { b"this" => (matches!(role, TokenRole::Operand), "an operand"), b"scoped" | b"ref" | b"out" | b"in" | b"readonly" => { @@ -3861,10 +3872,9 @@ mod tests { assert!( matches!( CsharpCode::get_op_type_with_code(node, lambda, Ancestors::known(chain)), - TokenRole::Unknown + TokenRole::Operator ), - "`{}` on a lambda is unbilled today, as it was before #1418; billing it is a \ - deliberate change and belongs in this assertion, not around it", + "`{}` on a lambda must be an operator, as its declaration spelling is (#1482)", String::from_utf8_lossy(text) ); }); @@ -3875,6 +3885,59 @@ mod tests { ); } + /// `static` and `async` in both of their kind spellings: the keyword + /// leaf under a method's `modifier` wrapper, and the childless + /// `modifier` the grammar aliases a lambda's keyword to (#1482). + /// + /// The lambdas take parenthesised parameters because the bare + /// `y => y` spelling is an `implicit_parameter`, which no Halstead + /// half bills, and that gap is not what this fixture measures. + #[cfg(feature = "csharp")] + const CSHARP_LAMBDA_MODIFIER_BOTH_SPELLINGS: &str = "class L { + static void N() { } + async void M() { } + void P() { + var a = static (int x) => x; + var b = async (int y) => y; + } +}"; + + /// A lambda's `static` / `async` is the same operator as the + /// declaration's, counted once in `n1` across both spellings + /// (#1482). + #[cfg(feature = "csharp")] + #[test] + fn csharp_lambda_modifiers_are_operators_1482() { + // Operators (n1 = 10, N1 = 25): class 1; {} 4 (the class body + // and three method bodies); static 2; async 2; void 3; () 5 + // (three method parameter lists and two lambda ones); int 2; + // = 2; ; 2; => 2. + // + // Operands (n2 = 8, N2 = 10): L, N, M, P, a, b once each; x 2 + // and y 2 (the parameter and the lambda body). `var` is an + // unclassified `implicit_type` wrapper. + // + // Before the fix this read (10, 23, 8, 10): the two lambda + // keywords were billed nowhere. Billing them without listing + // `Static` / `Async` in `CsharpCode::is_primitive` reads + // (12, 25, 8, 10) — each keyword split across the kind-keyed + // and the lexeme-keyed operator map. + assert_halstead_counts::( + CSHARP_LAMBDA_MODIFIER_BOTH_SPELLINGS, + "foo.cs", + [10, 25, 8, 10], + "csharp lambda modifiers", + ); + + assert_modifier_spellings_are_one_operator( + CSHARP_LAMBDA_MODIFIER_BOTH_SPELLINGS, + &[ + ("static", Csharp::Static as u16), + ("async", Csharp::Async as u16), + ], + ); + } + #[cfg(feature = "go")] #[test] fn go_operators_and_operands() { diff --git a/tests/corpus/csharp_test.rs b/tests/corpus/csharp_test.rs index 177f7b1f..68a3fb86 100644 --- a/tests/corpus/csharp_test.rs +++ b/tests/corpus/csharp_test.rs @@ -21,5 +21,5 @@ fn test_csharp() { .join("repositories") .join("big-code-analysis-output"); - compare_rca_output_with_files_under(&source_root, "csharp", &["*.cs"], &[], 6); + compare_rca_output_with_files_under(&source_root, "csharp", &["*.cs"], &[], 7); } diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index 0b653fcb..bdf73cf0 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit 0b653fcb691c0bd2cc78296437772b044ed94719 +Subproject commit bdf73cf01580e8dde76e1d0bc1e4130a9117079f From 2b41e73b42a2151b44b84c9a33368b2fcacad41b Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:08:25 -0700 Subject: [PATCH 14/48] fix(getter): bill overloaded operator names An overloaded operator's name was an operator only when the grammar spelled it with punctuation (`operator +`). C#'s `operator true` / `operator false` emit the keyword tokens, which were billed as operands, and C++'s `operator[]`, `operator()`, `operator""_x` and `operator co_await` emit whole tokens of their own that no arm listed, so they were billed nowhere. C#: the `True | False` arm gains an `OperatorDeclaration` parent guard beside the #1253 `BooleanLiteral` one. C++ and Mozcpp: `LPARENRPAREN`, `LBRACKRBRACK`, `DQUOTEDQUOTE`, `CoAwait` and the `operator` keyword are operators. `co_await` is also the keyword of a `co_await` expression. C and ObjC have none of these kinds. 254 DeepSpeech C++ snapshots move, all in Halstead operator counts and the fields derived from them (and mi); no operand or structural change. Fixes #1296 --- .rustfmt-bail-baseline.txt | 7 +- big-code-analysis-ast/src/getter/cpp.rs | 22 ++- big-code-analysis-ast/src/getter/csharp.rs | 13 +- big-code-analysis-ast/src/getter/mozcpp.rs | 6 +- big-code-analysis-book/src/metrics.md | 9 ++ src/metrics/halstead.rs | 143 +++++++++++++++----- tests/repositories/big-code-analysis-output | 2 +- 7 files changed, 158 insertions(+), 44 deletions(-) diff --git a/.rustfmt-bail-baseline.txt b/.rustfmt-bail-baseline.txt index 9f2d36b9..3656a936 100644 --- a/.rustfmt-bail-baseline.txt +++ b/.rustfmt-bail-baseline.txt @@ -125,8 +125,8 @@ big-code-analysis-ast/src/getter.rs 3 big-code-analysis-ast/src/getter/bash.rs 3 big-code-analysis-ast/src/getter/c.rs 3 -big-code-analysis-ast/src/getter/cpp.rs 5 -big-code-analysis-ast/src/getter/csharp.rs 8 +big-code-analysis-ast/src/getter/cpp.rs 4 +big-code-analysis-ast/src/getter/csharp.rs 9 big-code-analysis-ast/src/getter/elixir.rs 3 big-code-analysis-ast/src/getter/go.rs 1 big-code-analysis-ast/src/getter/groovy.rs 7 @@ -134,7 +134,7 @@ big-code-analysis-ast/src/getter/irules.rs 6 big-code-analysis-ast/src/getter/java.rs 5 big-code-analysis-ast/src/getter/kotlin.rs 4 big-code-analysis-ast/src/getter/lua.rs 3 -big-code-analysis-ast/src/getter/mozcpp.rs 5 +big-code-analysis-ast/src/getter/mozcpp.rs 4 big-code-analysis-ast/src/getter/perl.rs 3 big-code-analysis-ast/src/getter/php.rs 11 big-code-analysis-ast/src/getter/python.rs 7 @@ -142,7 +142,6 @@ big-code-analysis-ast/src/getter/ruby.rs 4 big-code-analysis-ast/src/getter/tcl.rs 6 big-code-analysis-ast/src/macros/mod.rs 14 enums/src/macros.rs 1 -src/metrics/abc/elixir.rs 5 src/metrics/cognitive/perl.rs 8 src/metrics/cyclomatic.rs 7 src/metrics/cyclomatic/irules.rs 2 diff --git a/big-code-analysis-ast/src/getter/cpp.rs b/big-code-analysis-ast/src/getter/cpp.rs index 81f22d65..46b84e5f 100644 --- a/big-code-analysis-ast/src/getter/cpp.rs +++ b/big-code-analysis-ast/src/getter/cpp.rs @@ -139,7 +139,27 @@ impl Getter for CppCode { // modifier has a distinct kind_id, so keying by kind_id (the default // `operators` store) keeps them distinct in n1 while `long long`'s // two `long` tokens correctly fold to one n1 entry but two N1 hits. - | Signed | Unsigned | Long | Short => TokenRole::Operator, + | Signed | Unsigned | Long | Short + // An overloaded operator's name (#1296). `operator +` already + // billed its `+` through the punctuation kind, but the grammar + // spells four overloadable names as whole tokens of their own + // — `[]`, `()`, the `""` of a user-defined literal and + // `co_await` — and none was in either arm, so + // `operator[]` and `operator()` billed nothing for the name + // that tells them apart. `Operator` is the `operator` keyword + // itself, which C# bills the same way. `CoAwait` is also the + // keyword of `co_await expr`, an operator there too. Each + // token occurs only under `operator_name`, `operator_cast` or + // `co_await_expression`, none of which is classified, so + // nothing bills one twice (grammar-dispatch section 5). + // + // `LPARENRPAREN` / `LBRACKRBRACK` are kinds of their own, not + // the `LPAREN` / `LBRACK` of a call or subscript, so declaring + // `operator[]` and applying `s[1]` are two `n1` entries that + // `bca ops` both renders `[]` — the same choice Ruby makes for + // its `def [](i)` method names. + | LPARENRPAREN | LBRACKRBRACK | DQUOTEDQUOTE | CoAwait | Operator + => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only // classified node in a character literal, so it bills one diff --git a/big-code-analysis-ast/src/getter/csharp.rs b/big-code-analysis-ast/src/getter/csharp.rs index c3fa305f..1e78f808 100644 --- a/big-code-analysis-ast/src/getter/csharp.rs +++ b/big-code-analysis-ast/src/getter/csharp.rs @@ -85,14 +85,15 @@ impl Getter for CsharpCode { // the text-keyed `n2` hid it (#1253). The leaf cannot simply // be dropped: the overloadable-operator list emits a bare // `true` / `false` under `operator_declaration` — the - // grammar's only unwrapped position — and suppressing that - // would leave two such declarations with identical Halstead - // vocabularies whenever their bodies match. That position - // keeps its pre-#1253 `Operand` classification here; whether - // an overloaded operator's *name* is better counted as an - // operator, as `operator +` already is, is #1296. + // grammar's only unwrapped position. There the keyword + // *names* an operator, exactly as the `+` of `operator +` + // does, so it is billed as one (#1296); C# merely spells two + // of its overloadable operators with keywords instead of + // punctuation. The fallback keeps any other unwrapped + // position an operand, the role the keyword has as a value. True | False => match ancestors.parent(node).map(|p| p.kind_id().into()) { Some(BooleanLiteral) => TokenRole::Unknown, + Some(OperatorDeclaration) => TokenRole::Operator, _ => TokenRole::Operand, }, // `this` is a self-reference everywhere (`this.x`, diff --git a/big-code-analysis-ast/src/getter/mozcpp.rs b/big-code-analysis-ast/src/getter/mozcpp.rs index 450366b4..1d274d4d 100644 --- a/big-code-analysis-ast/src/getter/mozcpp.rs +++ b/big-code-analysis-ast/src/getter/mozcpp.rs @@ -109,7 +109,11 @@ impl Getter for MozcppCode { // modifier has a distinct kind_id, so keying by kind_id (the default // `operators` store) keeps them distinct in n1 while `long long`'s // two `long` tokens correctly fold to one n1 entry but two N1 hits. - | Signed | Unsigned | Long | Short => TokenRole::Operator, + | Signed | Unsigned | Long | Short + // An overloaded operator's name, the `operator` keyword and + // `co_await` — the derivation is on the Cpp twin (#1296). + | LPARENRPAREN | LBRACKRBRACK | DQUOTEDQUOTE | CoAwait | Operator + => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only // classified node in a character literal, so it bills one diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index efb70f0e..fda7b6c4 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -711,6 +711,15 @@ tokens involved the same way they spell real operators: call `M(ref x)`, and `static (int x) => x` bills the same `static` as `static void M()`. A repeated operator keyword adds to `total_operators` but not to `unique_operators` (#1418, #1482). +- **An overloaded operator's name is an operator.** The `+` of + `operator +` was always one, and the names a grammar spells as whole + tokens now agree: C#'s `operator true` / `operator false`, and C++'s + `operator[]`, `operator()`, `operator""_x` and `operator co_await`. + The `operator` keyword is an operator in both languages, as is C++'s + `co_await` in an expression. C++'s `[]` and `()` names are tokens of + their own, so `bca ops` lists `operator[]`'s `[]` beside a + subscript's `[]` as a second operator, as Ruby does for `def [](i)` + (#1296). - **A named operator is one operand.** Elixir names an operator without applying it in a capture (`&==/2`, `&and/2`) or a remote call (`Kernel.||(a, b)`), and its grammar wraps the token in an diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 6732aef8..d1625c13 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -1089,12 +1089,13 @@ mod tests { #[test] fn cpp_dot_star_is_halstead_operator() { check_metrics::("struct S { void operator.*(int); };", "foo.cpp", |metric| { - // Unique operators with fix: {}, ;, (), int, void, .* + // Unique operators with fix: {}, ;, (), int, void, .*, and + // the `operator` keyword (#1296). // `.*` is the regression target — without the fix it - // falls through to `Unknown` and `u_operators` is 5. + // falls through to `Unknown` and `u_operators` is 6. // Unique operands: S let s = &metric.halstead; - assert_eq!(s.unique_operators(), 6); + assert_eq!(s.unique_operators(), 7); assert_eq!(s.unique_operands(), 1); }); } @@ -1116,12 +1117,13 @@ mod tests { "struct S { void operator->*(int); };", "foo.cpp", |metric| { - // Unique operators with fix: {}, ;, (), int, void, ->* + // Unique operators with fix: {}, ;, (), int, void, ->*, + // and the `operator` keyword (#1296). // `->*` is the regression target — without the fix it - // falls through to `Unknown` and `u_operators` is 5. + // falls through to `Unknown` and `u_operators` is 6. // Unique operands: S let s = &metric.halstead; - assert_eq!(s.unique_operators(), 6); + assert_eq!(s.unique_operators(), 7); assert_eq!(s.unique_operands(), 1); }, ); @@ -3339,33 +3341,45 @@ mod tests { #[cfg(feature = "csharp")] #[test] - fn csharp_boolean_keyword_outside_a_literal_still_counts() { - // Companion to the test above (#1253): the suppression fires on - // the *parent* kind, never on `True` / `False` alone. C#'s - // overloadable-operator list emits a bare `true` / `false` token - // with no `boolean_literal` wrapper — `operator_declaration` is - // the grammar's only such position — so a blanket exclusion - // would drop the operand that is the sole difference between - // `operator true` and `operator false`, leaving two such - // declarations with identical Halstead vocabularies whenever - // their bodies match. + fn csharp_operator_true_names_an_operator_not_a_value() { + // C#'s overloadable-operator list emits a bare `true` / `false` + // token with no `boolean_literal` wrapper — `operator_declaration` + // is the grammar's only such position. There the keyword names + // the operator being declared, as `+` does in `operator +`, so it + // is an operator (#1296); the companion test above pins the + // wrapped literal, which bills only its wrapper (#1253). // // Each declaration names one boolean and returns the other, so - // the fixture exercises both the guarded and the unguarded - // position for each keyword. - // - // Operands: `A` × 3 (class name, two parameter types), `a` × 2, - // `true` × 2 (operator name + literal), `false` × 2 (likewise) - // ⇒ n2 = 4, N2 = 9. A blanket exclusion gives N2 = 7; no guard - // at all restores the double count at N2 = 11. - check_metrics::( - "class A {\n public static bool operator true(A a) => false;\n public static bool operator false(A a) => true;\n}\n", - "foo.cs", - |metric| { - assert_eq!(metric.halstead.unique_operands(), 4); - assert_eq!(metric.halstead.total_operands(), 9); - }, - ); + // the fixture carries each keyword once as a name and once as a + // value. + // + // Operators (n1 = 11, N1 = 18): class 1, {} 1, public 2, + // static 2, bool 2, operator 2, true 1, false 1, () 2, => 2, + // ; 2. Operands (n2 = 4, N2 = 7): A × 3 (class name, two + // parameter types), a × 2, and the literals false and true once + // each. Before #1296 the two names were operands instead: + // (9, 16, 4, 9). A blanket exclusion of the leaf would give + // (9, 16, 4, 7), and no guard at all (11, 18, 4, 9). + let source = "class A {\n public static bool operator true(A a) => false;\n public static bool operator false(A a) => true;\n}\n"; + assert_halstead_counts::(source, "foo.cs", [11, 18, 4, 7], "operator true"); + + // `true` and `false` are each an operator *and* an operand here, + // once apiece: the name and the literal land in different halves. + let ops = ops_of::(source, "foo.cs"); + for keyword in ["true", "false"] { + for (half, list) in [("operators", &ops.operators), ("operands", &ops.operands)] { + assert_eq!( + list.iter().filter(|o| *o == keyword).count(), + 1, + "`{keyword}` must appear once among the {half}; they were {list:?}" + ); + } + } + + // The fallback arm: outside both positions — here a bare `true` + // under error recovery, the only other parent the grammar + // produces — the keyword keeps its role as a value. + assert_halstead_counts::("true\n", "foo.cs", [0, 0, 1, 1], "bare true"); } #[cfg(feature = "csharp")] @@ -9815,6 +9829,73 @@ f() { ); } + /// Every C++ overloaded-operator name the grammar spells as a token + /// of its own, plus the `co_await` expression and the conversion + /// operator's `operator` keyword (#1296). + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const CPP_OPERATOR_NAMES: &str = + "struct S { int operator[](int); void operator()(); bool operator co_await(); }; +long operator\"\"_x(unsigned long long); +S::operator bool() const; +S s; int a = s[1]; auto t = co_await s; +"; + + /// Asserts `CPP_OPERATOR_NAMES` bills each operator name, keyword and + /// `co_await` as an operator under parser `T`. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + #[track_caller] + fn assert_cpp_operator_names(label: &str) { + // Operators (n1 = 16, N1 = 39): `operator` 5 (four + // `operator_name`s and the `operator_cast`); the names `[]` 1, + // `()` 1, `""` 1; `co_await` 2 (the name and the expression); + // the `(` of a parameter list 5; `;` 9; `int` 3; `long` 3; + // `bool` 2; `=` 2; `{` 1; `void` 1; `unsigned` 1; `::` 1; the + // subscript `[` 1. `struct`, `auto` and `const` are in no arm. + // + // Operands (n2 = 6, N2 = 9): `s` 3, `S` 2 (the qualifier in + // `S::operator bool` is a namespace identifier, which #1096 + // leaves unbilled), `_x`, `a`, `1`, `t`. + // + // Before #1296: (11, 29, 6, 9), the five operator kinds billed + // nowhere. + assert_halstead_counts::(CPP_OPERATOR_NAMES, "ops.cpp", [16, 39, 6, 9], label); + + // `operator[]` / `operator()` are kinds of their own, so they sit + // beside the subscript `[` and the parameter-list `(` as a second + // `[]` / `()` entry — two operators, as Ruby's `def [](i)` is. + let ops = ops_of::(CPP_OPERATOR_NAMES, "ops.cpp"); + for (name, entries) in [ + ("operator", 1), + ("co_await", 1), + ("\"\"", 1), + ("[]", 2), + ("()", 2), + ] { + assert_eq!( + ops.operators.iter().filter(|o| *o == name).count(), + entries, + "{label}: `{name}` must be {entries} operator entry; operators were {:?}", + ops.operators + ); + assert!( + !ops.operands.iter().any(|o| o == name), + "{label}: `{name}` must not be an operand; operands were {:?}", + ops.operands + ); + } + } + + /// Regression for #1296: a C++ operator name is a Halstead operator, + /// as `operator +`'s `+` already was. One row per clone, so reverting + /// either arm fails its own row (grammar-dispatch section 11); Mozcpp + /// owns no file extension, so this row is its only coverage. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_operator_names_are_operators() { + assert_cpp_operator_names::("cpp"); + assert_cpp_operator_names::("mozcpp"); + } + /// Regression for #1361: a C++ `this` is a Halstead operand. /// /// `Cpp::This` / `Mozcpp::This` were in neither arm of diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index bdf73cf0..dfa8c2c8 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit bdf73cf01580e8dde76e1d0bc1e4130a9117079f +Subproject commit dfa8c2c8fba687557852dba025c26874eaaf21a5 From d2160e7ff61db3bc8ec3c14e8e33cfa26ed595dc Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:23:23 -0700 Subject: [PATCH 15/48] fix(getter/js-family): bill JSX text as operands `jsx_text` and `html_character_reference` were in no arm of the JavaScript, MozJS or TSX getters, so the text content of a JSX element reached neither Halstead half. A new `jsx:` argument to `impl_js_family_get_op_type!` bills element text as an operand, and a character reference as one when its parent is not a `string` (inside an attribute string it is already part of that operand). It also emits two byte-aware overrides: whitespace-only text (the same-line space between two tags) is `Unknown`, and the operand is keyed on the trimmed text so indentation cannot split identical text. `Checker::is_string` deliberately stays without `jsx_text`: `find string` reports delimited literals, the set the alterator flattens. TypeScript gets no `jsx:` argument; a test pins that its grammar never emits `jsx_text`. No corpus contains JSX. Fixes #1483 --- big-code-analysis-ast/src/getter.rs | 70 ++++++++- .../src/getter/javascript.rs | 3 + big-code-analysis-ast/src/getter/mozjs.rs | 3 + big-code-analysis-ast/src/getter/tsx.rs | 3 +- .../src/getter/typescript.rs | 3 + big-code-analysis-book/src/metrics.md | 8 + src/metrics/halstead.rs | 138 +++++++++++++++++- 7 files changed, 225 insertions(+), 3 deletions(-) diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index 34f6d3f7..69b462b7 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -127,6 +127,35 @@ macro_rules! get_operator { // the *field definition* `#x = 1` had never been counted at all — no // wrapper covered it — so this fixes that half too.) // +// `jsx: [, ]` (JavaScript, MozJS, TSX) bills +// the content of a JSX element (#1483). A `jsx_text` node is an +// operand, the way a string's contents are; it is childless and no +// node containing it (`jsx_element`) is classified, so nothing bills it +// twice. Two refinements need the source bytes and so live in the +// `get_op_type_with_code` / `get_operand_id` overrides the parameter +// also emits: +// +// * A whitespace-only `jsx_text` is `Unknown`. The scanner emits no +// token for whitespace that starts with a newline, but same-line +// spaces between tags (` `) are a node of their own, and +// billing them would make `N2` follow the source's layout. +// * The operand is keyed on the text with its surrounding whitespace +// trimmed. A token keeps the newlines and indentation around it, so +// `Start` at two indentation depths would otherwise be two distinct +// operands. This is the same shape as Kotlin's #454 narrowing. +// +// An `html_character_reference` (`&`) splits the text around it +// into siblings, so `a & b` is three operands: `a`, `&` and +// `b`. Each node is one vocabulary entry, and the entity denotes a +// character the surrounding text does not, so it is not folded into +// either neighbour. The same kind also sits inside a JSX attribute's +// `string`, which is already the operand, so it is billed only when +// its parent is not a string. +// +// `Checker::is_string` deliberately does not list `jsx_text`: `find +// string` reports delimited string literals, the set the alterator +// flattens (#283), and element text is undelimited markup content. +// // The `TemplateString` interpolation guard is shared verbatim (issue // #192): a bare `` `...` `` mirrors a `"..."` operand, but an // interpolated template must yield `Unknown` because its inner @@ -145,7 +174,8 @@ macro_rules! impl_js_family_get_op_type { $lang:ident, op_extras: [$($op_extra:ident),* $(,)?], operand_extras: [$($operand_extra:ident),* $(,)?] - $(, predefined_void: $predefined_type:ident)? $(,)? + $(, predefined_void: $predefined_type:ident)? + $(, jsx: [$jsx_text:ident, $jsx_entity:ident])? $(,)? ) => { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use $lang::*; @@ -262,9 +292,47 @@ macro_rules! impl_js_family_get_op_type { TemplateString => { Self::string_operand_type(node, &[TemplateSubstitution as u16]) } + $( + $jsx_text => TokenRole::Operand, + $jsx_entity + if !ancestors + .parent(node) + .is_some_and(|p| matches!(p.kind_id().into(), String | String2)) => + { + TokenRole::Operand + } + )? _ => TokenRole::Unknown, } } + + $( + fn get_op_type_with_code<'a>( + node: &Node<'a>, + code: &[u8], + ancestors: Ancestors<'a, '_>, + ) -> TokenRole { + if node.kind_id() == $lang::$jsx_text as u16 + && code[node.start_byte()..node.end_byte()].trim_ascii().is_empty() + { + return TokenRole::Unknown; + } + Self::get_op_type(node, ancestors) + } + + fn get_operand_id<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + ) -> &'a [u8] { + let text = &code[node.start_byte()..node.end_byte()]; + if node.kind_id() == $lang::$jsx_text as u16 { + text.trim_ascii() + } else { + text + } + } + )? }; } diff --git a/big-code-analysis-ast/src/getter/javascript.rs b/big-code-analysis-ast/src/getter/javascript.rs index fd3b7ea4..c6c1c787 100644 --- a/big-code-analysis-ast/src/getter/javascript.rs +++ b/big-code-analysis-ast/src/getter/javascript.rs @@ -54,10 +54,13 @@ impl Getter for JavascriptCode { // JSX closing / self-closing tag delimiters. See the macro's // `$op_extras` note: they are per-language extras because // TypeScript, which shares this macro, has no JSX (#1395). + // `jsx:` bills element text and its entities as operands, for the + // same reason (#1483). impl_js_family_get_op_type!( Javascript, op_extras: [OptionalChain, LTSLASH, SLASHGT], operand_extras: [Identifier2, String2], + jsx: [JsxText, HtmlCharacterReference], ); get_operator!(Javascript); diff --git a/big-code-analysis-ast/src/getter/mozjs.rs b/big-code-analysis-ast/src/getter/mozjs.rs index 8fa2f02b..9486d448 100644 --- a/big-code-analysis-ast/src/getter/mozjs.rs +++ b/big-code-analysis-ast/src/getter/mozjs.rs @@ -54,10 +54,13 @@ impl Getter for MozjsCode { // JSX closing / self-closing tag delimiters. See the macro's // `$op_extras` note: they are per-language extras because // TypeScript, which shares this macro, has no JSX (#1395). + // `jsx:` bills element text and its entities as operands, for the + // same reason (#1483). impl_js_family_get_op_type!( Mozjs, op_extras: [OptionalChain, LTSLASH, SLASHGT], operand_extras: [Identifier2, String2], + jsx: [JsxText, HtmlCharacterReference], ); get_operator!(Mozjs); diff --git a/big-code-analysis-ast/src/getter/tsx.rs b/big-code-analysis-ast/src/getter/tsx.rs index 6eaf084f..ad01d6ba 100644 --- a/big-code-analysis-ast/src/getter/tsx.rs +++ b/big-code-analysis-ast/src/getter/tsx.rs @@ -73,12 +73,13 @@ impl Getter for TsxCode { // `LTSLASH` / `SLASHGT` are the JSX closing and self-closing tag // delimiters. TSX is the only one of the two TypeScript grammars // with JSX, so the `.ts` invocation has no such variants to pass - // (#1395). + // (#1395), and the same holds for the `jsx:` text kinds (#1483). impl_js_family_get_op_type!( Tsx, op_extras: [QMARKDOT, PredefinedType, LTSLASH, SLASHGT], operand_extras: [Identifier2, String2], predefined_void: PredefinedType, + jsx: [JsxText, HtmlCharacterReference], ); get_operator!(Tsx); diff --git a/big-code-analysis-ast/src/getter/typescript.rs b/big-code-analysis-ast/src/getter/typescript.rs index a082d555..3027db51 100644 --- a/big-code-analysis-ast/src/getter/typescript.rs +++ b/big-code-analysis-ast/src/getter/typescript.rs @@ -72,6 +72,9 @@ impl Getter for TypescriptCode { // once (#1261). `Checker::is_string` no longer matches the keyword // either, so the #313 parity rationale is retired rather than // contradicted. + // + // No `jsx:` argument: the `.ts` grammar's enum carries `JsxText` in + // its shared externals, but no production emits it (#1483). impl_js_family_get_op_type!( Typescript, op_extras: [QMARKDOT, PredefinedType], diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index fda7b6c4..99b24c4b 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -711,6 +711,14 @@ tokens involved the same way they spell real operators: call `M(ref x)`, and `static (int x) => x` bills the same `static` as `static void M()`. A repeated operator keyword adds to `total_operators` but not to `unique_operators` (#1418, #1482). +- **JSX element text is an operand.** In JavaScript, Mozjs and TSX, + the `hi` of `

hi

` counts the way a string's contents do. It is + keyed on its text with leading and trailing whitespace trimmed, so a + line of text at two indentation depths is one operand, and the + spaces between two tags on one line count as nothing. Whitespace + *inside* a multi-line text is kept. A character reference (`&`) splits the text + into separate operands (`a & b` is three). Inside an attribute + string, the reference is part of that string's one operand (#1483). - **An overloaded operator's name is an operator.** The `+` of `operator +` was always one, and the names a grammar spells as whole tokens now agree: C#'s `operator true` / `operator false`, and C++'s diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index d1625c13..1626297c 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -472,7 +472,7 @@ implement_metric_trait!(Halstead, PreprocCode, CcommentCode); clippy::too_many_lines )] mod tests { - use std::collections::HashSet; + use std::collections::{BTreeSet, HashSet}; use std::path::PathBuf; use crate::test_support::{ast_has_kind_id, check_metrics_only_shim, for_each_node_with_chain}; @@ -512,12 +512,15 @@ mod tests { feature = "groovy", feature = "irules", feature = "java", + feature = "javascript", feature = "kotlin", feature = "mozcpp", + feature = "mozjs", feature = "objc", feature = "perl", feature = "ruby", feature = "tcl", + feature = "typescript", ))] fn ops_of(source: &str, file: &str) -> crate::ops::Ops { let path = PathBuf::from(file); @@ -1716,6 +1719,139 @@ mod tests { ); } + /// One JSX shape per row of #1483's decision: the source, the + /// expected `n2` / `N2`, and the deduplicated operand vocabulary. + /// Every row is a single element bound to `a`, so the operands + /// outside the element's content are always `a` plus the tag names. + #[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))] + const JSX_TEXT_CASES: &[(&str, &str, [u64; 2], &[&str])] = &[ + // Element text is one operand. N2: a, p x2 (both tags), hi. + ("text", "const a =

hi

;\n", [3, 4], &["a", "p", "hi"]), + // The same-line space between `` and `` is a jsx_text + // node of its own, and bills nothing. Billing it would key an + // empty operand: (5, 6). + ( + "whitespace", + "const a =

;\n", + [4, 5], + &["a", "p", "b", "i"], + ), + // `Start` at two indentation depths is one distinct operand, + // because the key is the trimmed text. Untrimmed: (5, 7). + ( + "indentation", + "const a = (\n

\n Start\n \n Start\n \n

\n);\n", + [4, 7], + &["a", "p", "span", "Start"], + ), + // An entity splits the text into three siblings, each an + // operand. Without the entity arm: (4, 5). + ( + "entity in text", + "const a =

x & y

;\n", + [5, 6], + &["a", "p", "x", "&", "y"], + ), + // Inside an attribute string the entity is part of the string + // operand and bills nothing of its own. Without the parent + // guard: (5, 5). + ( + "entity in attribute", + "const a =

;\n", + [4, 4], + &["a", "p", "title", "\"x & y\""], + ), + // An expression child is unchanged by #1483: its identifier was + // already an operand. + ( + "expression", + "const a =

{v}

;\n", + [3, 4], + &["a", "p", "v"], + ), + ]; + + /// Asserts every row of [`JSX_TEXT_CASES`] under parser `T`, through + /// the metrics store (`n2` / `N2`) and the `--ops` vocabulary. + #[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))] + #[track_caller] + fn assert_jsx_text_operands(file: &str) { + for (label, source, expected, vocabulary) in JSX_TEXT_CASES { + crate::test_support::check_func_space_only::( + source, + file, + &[crate::Metric::Halstead], + |space| { + let halstead = &space.metrics.halstead; + assert_eq!( + [halstead.unique_operands(), halstead.total_operands()], + *expected, + "{file}: {label}" + ); + }, + ); + let ops = ops_of::(source, file); + let got: BTreeSet<&str> = ops.operands.iter().map(String::as_str).collect(); + let want: BTreeSet<&str> = vocabulary.iter().copied().collect(); + assert_eq!(got, want, "{file}: {label} operand vocabulary"); + } + } + + /// Regression for #1483: JSX element text and its character + /// references are operands. One row per grammar, so reverting one + /// invocation's `jsx:` argument fails its own row. + #[cfg(all(feature = "javascript", feature = "mozjs", feature = "typescript"))] + #[test] + fn jsx_text_is_an_operand() { + // A whitespace-only node bills nothing, so no count can tell the + // "whitespace" row's fixture from one that stopped spelling the + // same-line space. Pin the node's presence directly. + let (_, source, _, _) = JSX_TEXT_CASES + .iter() + .find(|(label, ..)| *label == "whitespace") + .expect("the whitespace row exists"); + let parser = + JavascriptParser::new(source.as_bytes().to_vec(), &PathBuf::from("foo.jsx"), None); + assert!( + parser.root().preorder().any(|n| { + n.kind_id() == Javascript::JsxText as u16 + && source.as_bytes()[n.start_byte()..n.end_byte()] + .trim_ascii() + .is_empty() + }), + "the whitespace row must carry a whitespace-only jsx_text node" + ); + + assert_jsx_text_operands::("foo.jsx"); + assert_jsx_text_operands::("foo.js"); + assert_jsx_text_operands::("foo.tsx"); + } + + /// The `.ts` grammar's enum carries `JsxText` in the externals it + /// shares with TSX, but no production emits it, which is why the + /// TypeScript invocation passes no `jsx:` argument (#1483). Pinned + /// so a grammar that starts emitting it cannot go unbilled quietly. + #[cfg(feature = "typescript")] + #[test] + fn typescript_never_emits_jsx_text() { + let parser = TypescriptParser::new( + b"const a =

hi

;\nlet b = (x);\n".to_vec(), + &PathBuf::from("foo.ts"), + None, + ); + // The positive half: the parse reached the angle-bracket syntax + // TSX would read as JSX, so the absence below is a property of + // the grammar rather than of a parse that produced nothing. + assert!( + ast_has_kind_id(&parser, Typescript::TypeAssertion as u16), + "fixture lost the `(x)` type assertion" + ); + assert!( + !ast_has_kind_id(&parser, Typescript::JsxText as u16), + "the .ts grammar now emits jsx_text; give its invocation a `jsx:` argument" + ); + } + #[cfg(feature = "typescript")] #[test] fn typescript_template_string_plain_is_operand() { From e560ed805913f10538da127972df60c78e1ea4e1 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:32:15 -0700 Subject: [PATCH 16/48] fix(getter/csharp): bill a lambda's implicit parameter `x => x` puts its parameter in a childless `implicit_parameter` node, where `(x) => x` puts an `identifier` in a `parameter_list`. The C# getter listed only the latter, so a bare lambda parameter was billed in neither Halstead half: `_ => 0` lost `_` from the vocabulary entirely. Add `ImplicitParameter` to the operand arm. `nargs` already counts both spellings alike through `is_bare_param` (#1185), and Java's bare lambda parameter is a plain `identifier` that was already an operand. The csharp corpus snapshots anonymous.cs and linq.cs move in total_operands and its derived fields only. Fixes #1544 --- big-code-analysis-ast/src/getter/csharp.rs | 8 ++++- src/metrics/halstead.rs | 37 ++++++++++++++++++--- tests/repositories/big-code-analysis-output | 2 +- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/big-code-analysis-ast/src/getter/csharp.rs b/big-code-analysis-ast/src/getter/csharp.rs index 1e78f808..74bc6c0f 100644 --- a/big-code-analysis-ast/src/getter/csharp.rs +++ b/big-code-analysis-ast/src/getter/csharp.rs @@ -138,7 +138,13 @@ impl Getter for CsharpCode { // spells inheritance (`class D : B`) holds the base type's // identifier and no `base` keyword, so nothing bills this // text twice. - Identifier | Base + // + // `ImplicitParameter` is the parameter of `x => x`: a + // childless leaf where `(x) => x` puts an `identifier` inside + // a `parameter_list`, so the bare spelling dropped its name + // from both Halstead halves (#1544). Nothing containing it + // is classified. + Identifier | Base | ImplicitParameter | IntegerLiteral | RealLiteral | BooleanLiteral | NullLiteral | CharacterLiteral | StringLiteral | VerbatimStringLiteral | RawStringLiteral => TokenRole::Operand, diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 1626297c..e03c1424 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -4038,10 +4038,6 @@ mod tests { /// `static` and `async` in both of their kind spellings: the keyword /// leaf under a method's `modifier` wrapper, and the childless /// `modifier` the grammar aliases a lambda's keyword to (#1482). - /// - /// The lambdas take parenthesised parameters because the bare - /// `y => y` spelling is an `implicit_parameter`, which no Halstead - /// half bills, and that gap is not what this fixture measures. #[cfg(feature = "csharp")] const CSHARP_LAMBDA_MODIFIER_BOTH_SPELLINGS: &str = "class L { static void N() { } @@ -4052,6 +4048,39 @@ mod tests { } }"; + /// A C# lambda's bare parameter (`x => x`) is the operand its + /// parenthesised twin (`(x) => x`) bills (#1544). Each row is one + /// lambda in a method body; the twins differ only by the `()` the + /// parenthesised spelling writes, so the operand halves must match + /// and the operator halves differ by exactly that one `()`. + #[cfg(feature = "csharp")] + #[test] + fn csharp_implicit_lambda_parameter_is_an_operand() { + let wrap = |lambda: &str| { + format!("class C {{\n void M() {{\n var f = {lambda};\n }}\n}}\n") + }; + // Operators: class, {} x2, void, () x1 (M's parameter list), =, + // =>, ; — n1 = 7, N1 = 8 — plus the lambda's own `()` in the + // parenthesised spelling. Operands: C, M, f, and the parameter + // twice; `var` is an unclassified `implicit_type`. `_ => 0` + // names `_` once and adds the literal `0`. + // + // Before #1544 the bare rows read (7, 8, 4, 4) for both: `x` + // lost one occurrence, and `_` left the vocabulary entirely. + for (lambda, expected) in [ + ("x => x", [7, 8, 4, 5]), + ("(x) => x", [7, 9, 4, 5]), + ("_ => 0", [7, 8, 5, 5]), + ("(_) => 0", [7, 9, 5, 5]), + ] { + assert_halstead_counts::(&wrap(lambda), "foo.cs", expected, lambda); + } + + // The bare parameter is billed under its own text, not under + // the kind name, and as an operand only. + assert_keywords_are_operands_only::(&wrap("_ => 0"), "foo.cs", &["_"]); + } + /// A lambda's `static` / `async` is the same operator as the /// declaration's, counted once in `n1` across both spellings /// (#1482). diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index dfa8c2c8..256811a8 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit dfa8c2c8fba687557852dba025c26874eaaf21a5 +Subproject commit 256811a850866f321bca53d387722829606dbe4d From c60e9591d35d2bd6ba0155643a9722e60efc1686 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:41:42 -0700 Subject: [PATCH 17/48] refactor(metrics): move Elixir decision helpers out of npa The Elixir clause, guard and `with` predicates that `Cyclomatic` and `Abc` share lived in `npa::shared`, which they have nothing to do with, and pushed that file past the `loc.ploc` limit. Move them verbatim into `metrics::elixir_decisions` and import them by name at the two call sites. No behaviour change; the `npa/shared.rs` file baseline entry is gone. Fixes #1543 --- .bca-baseline.toml | 6 - src/metrics/abc/elixir.rs | 14 +- src/metrics/cyclomatic/elixir.rs | 16 +- src/metrics/elixir_decisions.rs | 469 ++++++++++++++++++++++++++++++ src/metrics/mod.rs | 2 + src/metrics/npa/shared.rs | 471 ------------------------------- 6 files changed, 488 insertions(+), 490 deletions(-) create mode 100644 src/metrics/elixir_decisions.rs diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 6cc5b49b..0701b103 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -1037,12 +1037,6 @@ qualified = "python_self_attr_name_bytes" metric = "nexits" value = 6.0 -[[entry]] -path = "src/metrics/npa/shared.rs" -qualified = "" -metric = "loc.ploc" -value = 558.0 - [[entry]] path = "src/ops.rs" qualified = "ops_inner" diff --git a/src/metrics/abc/elixir.rs b/src/metrics/abc/elixir.rs index 3aa23e78..300eefa4 100644 --- a/src/metrics/abc/elixir.rs +++ b/src/metrics/abc/elixir.rs @@ -11,6 +11,10 @@ use super::{Abc, Stats, count_boolean_slot, count_field_operands, last_operand, wrapped_operand}; use crate::lang_helpers::elixir::{elixir_applying_operator, elixir_call_keyword}; +use crate::metrics::elixir_decisions::{ + elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_sole_unguarded_pattern, + elixir_when_alternative, elixir_when_is_guard, +}; use crate::*; /// The grammar rule an applied binary operator hangs off. Matched by @@ -136,7 +140,7 @@ fn elixir_count_condition(condition: &Node, conditions: &mut f64) { // required — so the `if let`'s else is unreachable at the pin rather // than untested. fn elixir_count_guard(when_operator: &Node, conditions: &mut f64) { - if let Some(alternative) = npa::elixir_when_alternative(when_operator) { + if let Some(alternative) = elixir_when_alternative(when_operator) { elixir_count_condition(&alternative, conditions); } } @@ -160,7 +164,7 @@ fn elixir_count_guard(when_operator: &Node, conditions: &mut f64) { // match, and a guard pays its own slot through the `when` arm, as the // two `Cyclomatic` decisions it carries. fn elixir_count_clause(clause: &Node, conditions: &mut f64) { - match npa::elixir_sole_unguarded_pattern(clause) { + match elixir_sole_unguarded_pattern(clause) { Some(pattern) => elixir_count_condition(&pattern, conditions), None => *conditions += 1., } @@ -361,7 +365,7 @@ impl Abc for ElixirCode { // is not re-plumbed out of the predicate because that // predicate's whole job (§7) is to be one boolean the `Abc` // and `Cyclomatic` impls share. - E::When if npa::elixir_when_is_guard(node, code, ancestors) => { + E::When if elixir_when_is_guard(node, code, ancestors) => { if let Some(operator) = ancestors.parent(node) { elixir_count_guard(&operator, &mut stats.conditions); } @@ -447,7 +451,7 @@ impl Abc for ElixirCode { count_field_operands(&chain, elixir_count_condition, &mut stats.conditions); } } - E::StabClause if npa::elixir_clause_is_decision(node, code, ancestors) => { + E::StabClause if elixir_clause_is_decision(node, code, ancestors) => { elixir_count_clause(node, &mut stats.conditions); } // A `with`'s `pattern <- expr` clause is a pattern-match @@ -456,7 +460,7 @@ impl Abc for ElixirCode { // pattern is never an operator that scores itself, and a // guard on it (`{:ok, a} when g <- x`) pays its own slot // through the `when` arm, as a guarded clause's does. - E::LTDASH if npa::elixir_arrow_is_with_clause(node, code, ancestors) => { + E::LTDASH if elixir_arrow_is_with_clause(node, code, ancestors) => { stats.conditions += 1.; } _ => {} diff --git a/src/metrics/cyclomatic/elixir.rs b/src/metrics/cyclomatic/elixir.rs index 88b62aa9..2ee64e10 100644 --- a/src/metrics/cyclomatic/elixir.rs +++ b/src/metrics/cyclomatic/elixir.rs @@ -7,6 +7,10 @@ )] use super::*; +use crate::metrics::elixir_decisions::{ + elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_when_is_guard, + elixir_with_else_dispatches, +}; impl Cyclomatic for ElixirCode { // Elixir's control-flow constructs are not distinct grammar @@ -72,9 +76,7 @@ impl Cyclomatic for ElixirCode { // `case`. Guarded forms (`_ when g ->`, `true when g ->`) // and named discards (`_x ->`) are real decisions and // still count. - E::StabClause - if crate::metrics::npa::elixir_clause_is_decision(node, code, ancestors) => - { + E::StabClause if elixir_clause_is_decision(node, code, ancestors) => { stats.cyclomatic += 1.; } // A guard is a decision the construct it guards does not @@ -98,7 +100,7 @@ impl Cyclomatic for ElixirCode { // No double count (§5): the token fires once per `when`, // and the `binary_operator` that wraps it is not matched by // any arm here. - E::When if crate::metrics::npa::elixir_when_is_guard(node, code, ancestors) => { + E::When if elixir_when_is_guard(node, code, ancestors) => { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } @@ -109,9 +111,7 @@ impl Cyclomatic for ElixirCode { // alternatives of one dispatch, so modified does not // collapse them. A `for` generator's `<-` is excluded by // the gate: the `for` already pays its loop decision. - E::LTDASH - if crate::metrics::npa::elixir_arrow_is_with_clause(node, code, ancestors) => - { + E::LTDASH if elixir_arrow_is_with_clause(node, code, ancestors) => { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } @@ -152,7 +152,7 @@ impl Cyclomatic for ElixirCode { // collapses is its `else`, the dispatch over // the failed value — once, and only when that // dispatch picks between anything (#1535). - "with" if crate::metrics::npa::elixir_with_else_dispatches(node, code) => { + "with" if elixir_with_else_dispatches(node, code) => { stats.cyclomatic_modified += 1.; } _ => {} diff --git a/src/metrics/elixir_decisions.rs b/src/metrics/elixir_decisions.rs new file mode 100644 index 00000000..03f891fa --- /dev/null +++ b/src/metrics/elixir_decisions.rs @@ -0,0 +1,469 @@ +//! Elixir clause and guard predicates shared by the `Cyclomatic` and +//! `Abc` impls for `ElixirCode`. +//! +//! Elixir spells every control-flow construct as a `Call`, so which +//! `stab_clause`, `when` and `<-` nodes are decisions is a question of +//! position and source text rather than node kind. Both metrics must +//! answer it identically (grammar-dispatch §7), so the answer lives in +//! one module. + +use crate::lang_helpers::elixir::{elixir_call_keyword, elixir_is_method_macro}; +use crate::{Ancestors, Elixir, Node}; + +/// Whether `node` is a `binary_operator` applying `operator`. +/// +/// `binary_operator` carries three kind aliases at this pin so it is +/// matched by rule name rather than by enumerating ids +/// (grammar-dispatch §1); `when` and `<-` each have exactly one id. +fn elixir_is_binary_operator(node: &Node, operator: Elixir) -> bool { + const BINARY_OPERATOR: &str = "binary_operator"; + + node.kind() == BINARY_OPERATOR + && node + .child_by_field_name("operator") + .is_some_and(|token| token.kind_id() == operator as u16) +} + +/// Whether `node` is a `when` `binary_operator` — the shape a repeated +/// guard's alternatives nest through. +fn elixir_is_when_operator(node: &Node) -> bool { + elixir_is_binary_operator(node, Elixir::When) +} + +/// Whether the two ancestors `up` yields first are a `with` Call's +/// `arguments` and the Call itself — the position of a `with` clause. +/// +/// `arguments` carries five kind aliases at this pin, so it is matched +/// by rule name (grammar-dispatch §1). +fn elixir_is_with_argument<'a>(mut up: impl Iterator>, code: &'a [u8]) -> bool { + up.next() + .is_some_and(|arguments| arguments.kind() == "arguments") + && up + .next() + .is_some_and(|call| elixir_call_keyword(&call, code) == Some("with")) +} + +/// Whether `node`, a `<-` token, is the operator of one of a `with`'s +/// `pattern <- expr` clauses: a decision, since the value either +/// matches the pattern or short-circuits to the `else` clauses (#1535). +/// It is the shape a Rust `if let` or `?` has, both of which pay one +/// decision in standard and modified cyclomatic alike. +/// +/// Only a direct argument of a `with` qualifies. The same token is a +/// `for` comprehension's generator (`for x <- xs`), which filters +/// rather than short-circuits and is deliberately left alone — `for` +/// already pays one decision for its loop — and a bitstring +/// generator (`for <>`) sits inside a `bitstring`, not under +/// `arguments`. +/// +/// Position alone is the gate, so it needs no separate test that the +/// token is applied (#1534): in every shape probed where it is not — +/// named (`with &<-/2`) or stranded by error recovery (`with(<-)`, +/// `with a, <- b`) — it sits under an `operator_identifier`, whose +/// parent is never the `with`'s `arguments`. +/// +/// A clause wrapped in parentheses after a space (`with (a <- b) do`) +/// parses through a `block` and is not recognised; Elixir drops the +/// parentheses, so it is a clause, but the spelling is rare. +/// +/// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode` +/// (grammar-dispatch §7). +pub(super) fn elixir_arrow_is_with_clause<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> bool { + elixir_is_with_argument(ancestors.iter(node).skip(1).map(|(a, _)| a), code) +} + +/// The one alternative a guard's `when` token introduces. +/// +/// `head when a when b` parses right-associatively as +/// `head when (a when b)`, so each `when` operator owns exactly one +/// alternative: the nested operator's `left` where its `right` nests a +/// further `when`, and its own `right` otherwise. One alternative per +/// token is what holds the `Abc` slot count level with the `Cyclomatic` +/// decision count at every chain length (§8), and what stops a chain of +/// n alternatives being counted once per token that can see it (§5). +/// +/// The `?` is infallible at the pinned grammar — `binary_operator` +/// declares `right` required — and is spelled as an `Option` because +/// `AGENTS.md` bans `expect` outside tests. +pub(super) fn elixir_when_alternative<'a>(when_operator: &Node<'a>) -> Option> { + let right = when_operator.child_by_field_name("right")?; + if elixir_is_when_operator(&right) { + right.child_by_field_name("left") + } else { + Some(right) + } +} + +/// Whether a `when` operator token spells a real guard — a function +/// head's (`def f(x) when g do`) or a clause's (`x when g -> …`) — +/// rather than a typespec's `when` binding clause +/// (`@spec f(a) :: a when a: integer`), which is type syntax and no +/// decision at all. +/// +/// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode` so the +/// two cannot disagree about what a guard is (grammar-dispatch §7). +/// Elixir has no dedicated guard production — `x when g` is an ordinary +/// `binary_operator` — so the position it sits in is the only thing that +/// tells a guard from a typespec, and the allowlist below is that +/// position set at the pinned grammar: the `left` slot of a +/// `stab_clause` (`case` / `cond` / `fn` / `receive` / `with`'s `else` +/// / `try`'s handlers), the pattern of a `with` clause +/// (`{:ok, a} when is_integer(a) <- g(x)`, #1535), or an argument of a +/// definition Call that takes a guarded head. +/// +/// `arguments` carries five kind aliases at this pin and +/// `binary_operator` three, so both are matched by rule name rather +/// than by enumerating ids (grammar-dispatch §1). +/// +/// Repeated guards (`when a when b`) are an or-chain: Elixir tries each +/// `when` expression in turn and moves to the next when the previous one +/// is false *or raises*, so the construct carries one decision per +/// alternative, level with the `when a or b` spelling. They parse +/// right-associatively into nested `when` operators, and only the +/// outermost sits on the anchor — so a nested one is a guard too, and +/// the climb below walks the `when` operators between it and the anchor +/// before asking the position question. [`elixir_when_alternative`] +/// names the one alternative each token introduces, which is how the +/// matching `Abc` slot stays one-per-token rather than double counting +/// the chain (grammar-dispatch §5). +/// +/// The climb stops at the first non-`when` ancestor, so an ordinary +/// single guard pays no extra step and a chain pays one per alternative +/// it lists — a bound set by the guard, not by the tree's depth. +/// +/// Both `None`s the climb answers `false` to are unreachable at the +/// pin: a `when` *token*'s chain always holds at least the +/// `binary_operator` it belongs to, and that operator always sits under +/// something, because the Elixir root is `source` and no `when` +/// operator can be it. They are `is_some_and`s rather than `expect`s +/// because `AGENTS.md` bans the `expect` outside tests. +pub(super) fn elixir_when_is_guard<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> bool { + let mut chain = ancestors.iter(node).map(|(ancestor, _)| ancestor); + // The token's parent is the `when` operator node itself; the first + // ancestor above the chain of `when` operators is the position that + // decides. + chain.next().is_some_and(|mut operator| { + let mut above = chain.next(); + while let Some(enclosing) = above.filter(elixir_is_when_operator) { + operator = enclosing; + above = chain.next(); + } + above.is_some_and(|parent| elixir_is_guard_position(&operator, &parent, chain, code)) + }) +} + +/// Whether `parent` — the first ancestor above the chain of `when` +/// operators that `operator` tops — holds that chain in a guard +/// position; `up` yields the ancestors above `parent`. The second step +/// of [`elixir_when_is_guard`], after the climb. +fn elixir_is_guard_position<'a>( + operator: &Node<'a>, + parent: &Node<'a>, + mut up: impl Iterator>, + code: &'a [u8], +) -> bool { + use Elixir as E; + + const ARGUMENTS: &str = "arguments"; + + if parent.kind_id() == E::StabClause as u16 { + return parent + .child_by_field_name("left") + .is_some_and(|left| left.id() == operator.id()); + } + // A guard on the pattern of a `with` clause. `when` binds tighter + // than `<-`, so it parses as the clause's `left`; a `when` on the + // right would be a guard outside any guard position, which Elixir + // rejects, so the side is not checked. The guard of a `for` + // generator (`for x when x > 1 <- xs`) is a filter, which the + // comprehension's own decision already covers exactly as a bare + // filter (`for x <- xs, x > 1`) is — so it is not a guard here. + if elixir_is_binary_operator(parent, E::LTDASH) { + return elixir_is_with_argument(up, code); + } + parent.kind() == ARGUMENTS + && up.next().is_some_and(|call| { + elixir_call_keyword(&call, code).is_some_and(|keyword| { + elixir_is_method_macro(keyword) || matches!(keyword, "defguard" | "defguardp") + }) + }) +} + +/// Returns the sole pattern of a `stab_clause` whose left-hand side +/// carries no `when` guard, or `None` otherwise. +/// +/// The grammar's `left` field is an `arguments` node for a plain +/// pattern list, but a guarded clause (`_ when g ->`) re-shapes it +/// into a `binary_operator` wrapping the patterns and the guard — so +/// checking the field's kind answers "is there a guard" for free. The +/// kind is compared as a string because the grammar aliases several +/// internal rules to `arguments` with distinct kind ids +/// (`Arguments2`..`Arguments5`); grammar-dispatch §1 prefers the one +/// string comparison over enumerating them. A zero-arity clause carries +/// no `left` field at all (`fn -> … end`), which the same `filter` folds +/// into "no plain pattern list" — the two spellings of "nothing to +/// inspect" are one question, so they share one exit. Named-children +/// filtering skips the anonymous `(` `)` tokens a parenthesised clause +/// head carries, so an empty pair (`fn () -> … end`) also yields zero +/// patterns. Clauses with zero or several patterns return `None`. +pub(super) fn elixir_sole_unguarded_pattern<'a>(node: &Node<'a>) -> Option> { + let left = node + .child_by_field_name("left") + .filter(|left| left.kind() == "arguments")?; + let mut patterns = left.children().filter(Node::is_named); + let sole = patterns.next()?; + patterns.next().is_none().then_some(sole) +} + +/// Whether `pattern` is a bare `_`. Elixir has no wildcard token — `_` +/// parses as an ordinary `identifier` — so the bytes decide. +fn elixir_is_bare_wildcard(pattern: &Node, code: &[u8]) -> bool { + pattern.kind_id() == Elixir::Identifier as u16 && pattern.utf8_text(code) == Some("_") +} + +/// Returns `true` when `node` is a construct's free default clause: +/// a bare `_ ->` catch-all outside a head-skipping construct (an +/// anonymous fn or a `for … reduce:`), or an unguarded `true ->` +/// directly under a `cond`. +/// +/// Both shapes hinge on the clause's sole unguarded pattern, and +/// [`elixir_sole_unguarded_pattern`]'s child scan allocates a cursor +/// (the #1112 malloc) — so the pattern is extracted once here and the +/// two shapes branch on its (kind, text), instead of every counted +/// `stab_clause` paying the extraction twice through two independent +/// predicates. +/// +/// Bare `_ ->`: Elixir has no dedicated wildcard token — `_` parses +/// as an ordinary `identifier` — so the bytes decide (grammar-dispatch +/// §10). A named discard (`_x ->`) binds a value the body can read +/// and keeps counting, matching Rust's bare-`_`-only `MatchArm` rule; +/// guarded wildcards never reach the text check because +/// [`elixir_sole_unguarded_pattern`] rejects them. The exclusion does +/// NOT apply when `skips_head` — the clause's container is an +/// `anonymous_function` or a `for … reduce:`: such a multi-clause +/// dispatch is like `case` — n clauses are n−1 decisions — and its free +/// base path is already granted by the head-clause skip (#776, #1535), +/// so excluding a trailing `_ ->` too would leave +/// `fn 0 -> :a; _ -> :b end` at zero decisions while the identical +/// `case` reports one. +/// +/// `true ->` under `cond`: the exclusion is *shape-based* — any +/// unguarded `true ->` whose parent is the `do_block` of a `Call` +/// spelling `cond`, whatever the arm's position. That deliberately +/// matches the sibling convention: Rust's bare-`_` `MatchArm` +/// exclusion is equally position-blind. The same clause under `case` +/// is an ordinary boolean pattern match, so the exclusion is anchored +/// to the owning construct (grammar-dispatch §8); a guarded +/// `true when g ->` is a real decision and never reaches the +/// container check. +fn elixir_is_default_clause<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, + skips_head: bool, +) -> bool { + use Elixir as E; + + let Some(pattern) = elixir_sole_unguarded_pattern(node) else { + return false; + }; + match pattern.kind_id().into() { + // Bare `_` catch-all — free everywhere except where the + // head-clause skip already provides the free base path. + E::Identifier => !skips_head && elixir_is_bare_wildcard(&pattern, code), + // Unguarded `true` — free only as `cond`'s designated + // default; the O(1) parent/grandparent checks run after the + // text compare so non-`true` booleans bail early. + E::Boolean => { + pattern.utf8_text(code) == Some("true") + && elixir_do_section_call(node, code, ancestors) + .is_some_and(|call| elixir_call_keyword(&call, code) == Some("cond")) + } + _ => false, + } +} + +/// Whether `node`'s clauses-owner grants a free head clause: an +/// `anonymous_function` (#776), or a `for … reduce:` comprehension +/// (#1535), whose accumulator clauses are an anonymous fn in all but +/// spelling — a single `acc -> …` always matches, and only the 2nd+ +/// clauses dispatch. +/// +/// A `for` holds clauses in its `do` section only under `reduce:`; +/// without it the section is a plain body. So the Call's keyword alone +/// identifies the shape, with no need to find the `reduce:` pair. +fn elixir_clauses_skip_head<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> bool { + ancestors.parent_has_kind(node, Elixir::AnonymousFunction as u16) + || elixir_do_section_call(node, code, ancestors) + .is_some_and(|call| elixir_call_keyword(&call, code) == Some("for")) +} + +/// Returns `true` when `node` is its parent's first `stab_clause` — +/// the head clause of a construct [`elixir_clauses_skip_head`] names. +/// +/// The parent is the `anonymous_function` (`fn stab_clause+ end`), the +/// `do_block` (`do stab_clause+ end`) or the keyword form's +/// parenthesised `block`; in each the first child whose kind is +/// `stab_clause` (skipping the `fn` / `do` / `(` token) is the head. +fn elixir_is_first_clause<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + ancestors + .parent(node) + .and_then(|parent| { + parent + .children() + .find(|child| child.kind_id() == Elixir::StabClause as u16) + }) + .is_some_and(|first| first.id() == node.id()) +} + +/// Whether `node` (a `stab_clause`) is one decision of the construct +/// that owns it: every clause of a `case` / `cond` / `with`'s `else` / +/// `receive` / `try` handler and every clause after a multi-clause +/// `fn`'s first, except the construct's free default clause +/// ([`elixir_is_default_clause`]). +/// +/// An anonymous fn's head clause is the closure's definition, not a +/// dispatch: the closure opens its own function space whose base path +/// already covers it, so counting it too would score a trivial +/// `fn x -> x end` as a decision (#776). A `for … reduce:`'s first +/// accumulator clause is skipped for the same reason: the `for` already +/// pays its one decision, and a single `acc -> …` clause always matches +/// (#1535). +/// +/// The parent is allowlisted because the grammar puts a `stab_clause` +/// in one more position: a parenthesised `block`, which is how a +/// typespec spells a function type (`@spec f((any -> any)) :: list`). +/// That is type syntax and branches on nothing, the same reason +/// [`elixir_when_is_guard`] excludes a typespec's `when` (#1531). The +/// keyword form of a clause construct (`case(x, do: (1 -> :a; …))`) +/// holds its clauses in the same parenthesised `block`, so a `block` +/// counts when it is the value of a section pair +/// ([`elixir_is_section_pair`]). +/// +/// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode`, which +/// both score a clause construct per clause and must agree on which +/// clauses those are (grammar-dispatch §7, #1531). +pub(super) fn elixir_clause_is_decision<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> bool { + use Elixir as E; + + let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); + let in_section = up.next().is_some_and(|parent| { + matches!( + parent.kind_id().into(), + E::DoBlock + | E::ElseBlock + | E::AfterBlock + | E::RescueBlock + | E::CatchBlock + | E::AnonymousFunction + ) || (parent.kind_id() == E::Block as u16 + && up + .next() + .is_some_and(|pair| elixir_is_section_pair(&pair, code, SECTION_KEYS))) + }); + if !in_section { + return false; + } + let skips_head = elixir_clauses_skip_head(node, code, ancestors); + if skips_head && elixir_is_first_clause(node, ancestors) { + return false; + } + !elixir_is_default_clause(node, code, ancestors, skips_head) +} + +/// Whether a `with` Call's `else` clauses dispatch: whether at least one +/// of them is a decision rather than the free bare `_ ->` default. +/// +/// Modified cyclomatic collapses a clause construct's arms into one +/// decision, which for `case` / `cond` / `try` is the container paying +/// once. A `with`'s decisions are its `<-` clauses, each paid on its own +/// in both tiers as a Rust `if let` is; its `else` is the dispatch over +/// the failed value, which collapses into one decision exactly when it +/// holds one — so `else _ -> 0`, which picks nothing, stays free in +/// modified as it is in standard, like Rust's `if let … else` (#1535). +/// Spelled in either form: the block form's `else_block`, or the +/// keyword form's `else:` pair whose value is a parenthesised `block`. +/// +/// In a `with`'s `else` the default-clause rule reduces to the bare +/// `_`: `true ->` is free only under a `cond`, and the head skip only +/// under a `fn` or `reduce:`, so [`elixir_clause_is_decision`] agrees +/// clause by clause with the test below. +pub(super) fn elixir_with_else_dispatches(call: &Node, code: &[u8]) -> bool { + use Elixir as E; + + call.children().any(|child| { + let section = match child.kind_id().into() { + E::DoBlock => child + .children() + .find(|c| c.kind_id() == E::ElseBlock as u16), + _ if child.kind() == "arguments" => child + .children() + .filter(|c| c.kind_id() == E::Keywords as u16) + .flat_map(|keywords| keywords.children()) + .find(|pair| elixir_is_section_pair(pair, code, &["else:"])) + .and_then(|pair| pair.child_by_field_name("value")), + _ => None, + }; + section.is_some_and(|section| { + section.children().any(|clause| { + clause.kind_id() == E::StabClause as u16 + && !elixir_sole_unguarded_pattern(&clause) + .is_some_and(|pattern| elixir_is_bare_wildcard(&pattern, code)) + }) + }) + }) +} + +/// The keyword spellings of the clause sections a `do … end` block +/// spells as `do_block` / `else_block` / `after_block` / `rescue_block` +/// / `catch_block`. +const SECTION_KEYS: &[&str] = &["do:", "else:", "after:", "rescue:", "catch:"]; + +/// Whether `pair` is a keyword-form section (`do: (…)`) whose key is one +/// of `keys`. The key token's text carries its colon and the whitespace +/// after it, so it is trimmed before comparing. +fn elixir_is_section_pair(pair: &Node, code: &[u8], keys: &[&str]) -> bool { + pair.kind_id() == Elixir::Pair as u16 + && pair + .child_by_field_name("key") + .and_then(|key| key.utf8_text(code)) + .is_some_and(|key| keys.contains(&key.trim_end())) +} + +/// The `Call` whose `do` section holds `node` (a `stab_clause`): the +/// block form's `do_block` (whose parent is the call) or the keyword +/// form's `do: (…)` block (`block` → `pair` → `keywords` → `arguments` +/// → call). The caller asks the Call's keyword (`cond`, `for`). +fn elixir_do_section_call<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> Option> { + use Elixir as E; + + let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); + match up.next().map(|parent| parent.kind_id().into()) { + Some(E::DoBlock) => up.next(), + Some(E::Block) => up + .next() + .filter(|pair| elixir_is_section_pair(pair, code, &["do:"])) + .and_then(|_| up.nth(2)), + _ => None, + } +} diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index b9f9d0f4..30c75806 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -33,6 +33,8 @@ pub mod tokens; /// Weighted Methods per Class. pub mod wmc; +mod elixir_decisions; + // Gated on the union of every feature any test in the module names — // the `FIXTURES` rows plus the `bash` / `lua` / `c` row set that // `a_language_with_no_member_construct_emits_neither_block` uses. The diff --git a/src/metrics/npa/shared.rs b/src/metrics/npa/shared.rs index 60888332..cbd85780 100644 --- a/src/metrics/npa/shared.rs +++ b/src/metrics/npa/shared.rs @@ -668,477 +668,6 @@ pub(crate) fn ruby_in_clause_counts(in_clause: &Node, source: &[u8]) -> bool { }) } -/// Whether `node` is a `binary_operator` applying `operator`. -/// -/// `binary_operator` carries three kind aliases at this pin so it is -/// matched by rule name rather than by enumerating ids -/// (grammar-dispatch §1); `when` and `<-` each have exactly one id. -fn elixir_is_binary_operator(node: &Node, operator: Elixir) -> bool { - const BINARY_OPERATOR: &str = "binary_operator"; - - node.kind() == BINARY_OPERATOR - && node - .child_by_field_name("operator") - .is_some_and(|token| token.kind_id() == operator as u16) -} - -/// Whether `node` is a `when` `binary_operator` — the shape a repeated -/// guard's alternatives nest through. -fn elixir_is_when_operator(node: &Node) -> bool { - elixir_is_binary_operator(node, Elixir::When) -} - -/// Whether the two ancestors `up` yields first are a `with` Call's -/// `arguments` and the Call itself — the position of a `with` clause. -/// -/// `arguments` carries five kind aliases at this pin, so it is matched -/// by rule name (grammar-dispatch §1). -fn elixir_is_with_argument<'a>(mut up: impl Iterator>, code: &'a [u8]) -> bool { - up.next() - .is_some_and(|arguments| arguments.kind() == "arguments") - && up.next().is_some_and(|call| { - crate::lang_helpers::elixir::elixir_call_keyword(&call, code) == Some("with") - }) -} - -/// Whether `node`, a `<-` token, is the operator of one of a `with`'s -/// `pattern <- expr` clauses: a decision, since the value either -/// matches the pattern or short-circuits to the `else` clauses (#1535). -/// It is the shape a Rust `if let` or `?` has, both of which pay one -/// decision in standard and modified cyclomatic alike. -/// -/// Only a direct argument of a `with` qualifies. The same token is a -/// `for` comprehension's generator (`for x <- xs`), which filters -/// rather than short-circuits and is deliberately left alone — `for` -/// already pays one decision for its loop — and a bitstring -/// generator (`for <>`) sits inside a `bitstring`, not under -/// `arguments`. -/// -/// Position alone is the gate, so it needs no separate test that the -/// token is applied (#1534): in every shape probed where it is not — -/// named (`with &<-/2`) or stranded by error recovery (`with(<-)`, -/// `with a, <- b`) — it sits under an `operator_identifier`, whose -/// parent is never the `with`'s `arguments`. -/// -/// A clause wrapped in parentheses after a space (`with (a <- b) do`) -/// parses through a `block` and is not recognised; Elixir drops the -/// parentheses, so it is a clause, but the spelling is rare. -/// -/// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode` -/// (grammar-dispatch §7). -pub(crate) fn elixir_arrow_is_with_clause<'a>( - node: &Node<'a>, - code: &'a [u8], - ancestors: Ancestors<'a, '_>, -) -> bool { - elixir_is_with_argument(ancestors.iter(node).skip(1).map(|(a, _)| a), code) -} - -/// The one alternative a guard's `when` token introduces. -/// -/// `head when a when b` parses right-associatively as -/// `head when (a when b)`, so each `when` operator owns exactly one -/// alternative: the nested operator's `left` where its `right` nests a -/// further `when`, and its own `right` otherwise. One alternative per -/// token is what holds the `Abc` slot count level with the `Cyclomatic` -/// decision count at every chain length (§8), and what stops a chain of -/// n alternatives being counted once per token that can see it (§5). -/// -/// The `?` is infallible at the pinned grammar — `binary_operator` -/// declares `right` required — and is spelled as an `Option` because -/// `AGENTS.md` bans `expect` outside tests. -pub(crate) fn elixir_when_alternative<'a>(when_operator: &Node<'a>) -> Option> { - let right = when_operator.child_by_field_name("right")?; - if elixir_is_when_operator(&right) { - right.child_by_field_name("left") - } else { - Some(right) - } -} - -/// Whether a `when` operator token spells a real guard — a function -/// head's (`def f(x) when g do`) or a clause's (`x when g -> …`) — -/// rather than a typespec's `when` binding clause -/// (`@spec f(a) :: a when a: integer`), which is type syntax and no -/// decision at all. -/// -/// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode` so the -/// two cannot disagree about what a guard is (grammar-dispatch §7). -/// Elixir has no dedicated guard production — `x when g` is an ordinary -/// `binary_operator` — so the position it sits in is the only thing that -/// tells a guard from a typespec, and the allowlist below is that -/// position set at the pinned grammar: the `left` slot of a -/// `stab_clause` (`case` / `cond` / `fn` / `receive` / `with`'s `else` -/// / `try`'s handlers), the pattern of a `with` clause -/// (`{:ok, a} when is_integer(a) <- g(x)`, #1535), or an argument of a -/// definition Call that takes a guarded head. -/// -/// `arguments` carries five kind aliases at this pin and -/// `binary_operator` three, so both are matched by rule name rather -/// than by enumerating ids (grammar-dispatch §1). -/// -/// Repeated guards (`when a when b`) are an or-chain: Elixir tries each -/// `when` expression in turn and moves to the next when the previous one -/// is false *or raises*, so the construct carries one decision per -/// alternative, level with the `when a or b` spelling. They parse -/// right-associatively into nested `when` operators, and only the -/// outermost sits on the anchor — so a nested one is a guard too, and -/// the climb below walks the `when` operators between it and the anchor -/// before asking the position question. [`elixir_when_alternative`] -/// names the one alternative each token introduces, which is how the -/// matching `Abc` slot stays one-per-token rather than double counting -/// the chain (grammar-dispatch §5). -/// -/// The climb stops at the first non-`when` ancestor, so an ordinary -/// single guard pays no extra step and a chain pays one per alternative -/// it lists — a bound set by the guard, not by the tree's depth. -/// -/// Both `return false` guards below are unreachable at the pin, not -/// untested: a `when` *token*'s chain always holds at least the -/// `binary_operator` it belongs to, and that operator always sits under -/// something, because the Elixir root is `source` and no `when` -/// operator can be it. They stay as defensive `else` arms because -/// `AGENTS.md` bans the `expect` that would replace them. -pub(crate) fn elixir_when_is_guard<'a>( - node: &Node<'a>, - code: &'a [u8], - ancestors: Ancestors<'a, '_>, -) -> bool { - let mut chain = ancestors.iter(node); - // The token's parent is the `when` operator node itself; the first - // ancestor above the chain of `when` operators is the position that - // decides. - let Some((mut operator, _)) = chain.next() else { - return false; - }; - let mut above = chain.next(); - while let Some((enclosing, _)) = above.filter(|(ancestor, _)| elixir_is_when_operator(ancestor)) - { - operator = enclosing; - above = chain.next(); - } - let Some((parent, _)) = above else { - return false; - }; - elixir_is_guard_position( - &operator, - &parent, - chain.map(|(ancestor, _)| ancestor), - code, - ) -} - -/// Whether `parent` — the first ancestor above the chain of `when` -/// operators that `operator` tops — holds that chain in a guard -/// position; `up` yields the ancestors above `parent`. The second step -/// of [`elixir_when_is_guard`], after the climb. -fn elixir_is_guard_position<'a>( - operator: &Node<'a>, - parent: &Node<'a>, - mut up: impl Iterator>, - code: &'a [u8], -) -> bool { - use Elixir as E; - - const ARGUMENTS: &str = "arguments"; - - if parent.kind_id() == E::StabClause as u16 { - return parent - .child_by_field_name("left") - .is_some_and(|left| left.id() == operator.id()); - } - // A guard on the pattern of a `with` clause. `when` binds tighter - // than `<-`, so it parses as the clause's `left`; a `when` on the - // right would be a guard outside any guard position, which Elixir - // rejects, so the side is not checked. The guard of a `for` - // generator (`for x when x > 1 <- xs`) is a filter, which the - // comprehension's own decision already covers exactly as a bare - // filter (`for x <- xs, x > 1`) is — so it is not a guard here. - if elixir_is_binary_operator(parent, E::LTDASH) { - return elixir_is_with_argument(up, code); - } - parent.kind() == ARGUMENTS - && up.next().is_some_and(|call| { - crate::lang_helpers::elixir::elixir_call_keyword(&call, code).is_some_and(|keyword| { - crate::lang_helpers::elixir::elixir_is_method_macro(keyword) - || matches!(keyword, "defguard" | "defguardp") - }) - }) -} - -/// Returns the sole pattern of a `stab_clause` whose left-hand side -/// carries no `when` guard, or `None` otherwise. -/// -/// The grammar's `left` field is an `arguments` node for a plain -/// pattern list, but a guarded clause (`_ when g ->`) re-shapes it -/// into a `binary_operator` wrapping the patterns and the guard — so -/// checking the field's kind answers "is there a guard" for free. The -/// kind is compared as a string because the grammar aliases several -/// internal rules to `arguments` with distinct kind ids -/// (`Arguments2`..`Arguments5`); grammar-dispatch §1 prefers the one -/// string comparison over enumerating them. A zero-arity clause carries -/// no `left` field at all (`fn -> … end`), which the same `filter` folds -/// into "no plain pattern list" — the two spellings of "nothing to -/// inspect" are one question, so they share one exit. Named-children -/// filtering skips the anonymous `(` `)` tokens a parenthesised clause -/// head carries, so an empty pair (`fn () -> … end`) also yields zero -/// patterns. Clauses with zero or several patterns return `None`. -pub(crate) fn elixir_sole_unguarded_pattern<'a>(node: &Node<'a>) -> Option> { - let left = node - .child_by_field_name("left") - .filter(|left| left.kind() == "arguments")?; - let mut patterns = left.children().filter(Node::is_named); - let sole = patterns.next()?; - patterns.next().is_none().then_some(sole) -} - -/// Whether `pattern` is a bare `_`. Elixir has no wildcard token — `_` -/// parses as an ordinary `identifier` — so the bytes decide. -fn elixir_is_bare_wildcard(pattern: &Node, code: &[u8]) -> bool { - pattern.kind_id() == Elixir::Identifier as u16 && pattern.utf8_text(code) == Some("_") -} - -/// Returns `true` when `node` is a construct's free default clause: -/// a bare `_ ->` catch-all outside a head-skipping construct (an -/// anonymous fn or a `for … reduce:`), or an unguarded `true ->` -/// directly under a `cond`. -/// -/// Both shapes hinge on the clause's sole unguarded pattern, and -/// [`elixir_sole_unguarded_pattern`]'s child scan allocates a cursor -/// (the #1112 malloc) — so the pattern is extracted once here and the -/// two shapes branch on its (kind, text), instead of every counted -/// `stab_clause` paying the extraction twice through two independent -/// predicates. -/// -/// Bare `_ ->`: Elixir has no dedicated wildcard token — `_` parses -/// as an ordinary `identifier` — so the bytes decide (grammar-dispatch -/// §10). A named discard (`_x ->`) binds a value the body can read -/// and keeps counting, matching Rust's bare-`_`-only `MatchArm` rule; -/// guarded wildcards never reach the text check because -/// [`elixir_sole_unguarded_pattern`] rejects them. The exclusion does -/// NOT apply when `skips_head` — the clause's container is an -/// `anonymous_function` or a `for … reduce:`: such a multi-clause -/// dispatch is like `case` — n clauses are n−1 decisions — and its free -/// base path is already granted by the head-clause skip (#776, #1535), -/// so excluding a trailing `_ ->` too would leave -/// `fn 0 -> :a; _ -> :b end` at zero decisions while the identical -/// `case` reports one. -/// -/// `true ->` under `cond`: the exclusion is *shape-based* — any -/// unguarded `true ->` whose parent is the `do_block` of a `Call` -/// spelling `cond`, whatever the arm's position. That deliberately -/// matches the sibling convention: Rust's bare-`_` `MatchArm` -/// exclusion is equally position-blind. The same clause under `case` -/// is an ordinary boolean pattern match, so the exclusion is anchored -/// to the owning construct (grammar-dispatch §8); a guarded -/// `true when g ->` is a real decision and never reaches the -/// container check. -fn elixir_is_default_clause<'a>( - node: &Node<'a>, - code: &'a [u8], - ancestors: Ancestors<'a, '_>, - skips_head: bool, -) -> bool { - use Elixir as E; - - let Some(pattern) = elixir_sole_unguarded_pattern(node) else { - return false; - }; - match pattern.kind_id().into() { - // Bare `_` catch-all — free everywhere except where the - // head-clause skip already provides the free base path. - E::Identifier => !skips_head && elixir_is_bare_wildcard(&pattern, code), - // Unguarded `true` — free only as `cond`'s designated - // default; the O(1) parent/grandparent checks run after the - // text compare so non-`true` booleans bail early. - E::Boolean => { - pattern.utf8_text(code) == Some("true") - && elixir_do_section_call(node, code, ancestors).is_some_and(|call| { - crate::lang_helpers::elixir::elixir_call_keyword(&call, code) == Some("cond") - }) - } - _ => false, - } -} - -/// Whether `node`'s clauses-owner grants a free head clause: an -/// `anonymous_function` (#776), or a `for … reduce:` comprehension -/// (#1535), whose accumulator clauses are an anonymous fn in all but -/// spelling — a single `acc -> …` always matches, and only the 2nd+ -/// clauses dispatch. -/// -/// A `for` holds clauses in its `do` section only under `reduce:`; -/// without it the section is a plain body. So the Call's keyword alone -/// identifies the shape, with no need to find the `reduce:` pair. -fn elixir_clauses_skip_head<'a>( - node: &Node<'a>, - code: &'a [u8], - ancestors: Ancestors<'a, '_>, -) -> bool { - ancestors.parent_has_kind(node, Elixir::AnonymousFunction as u16) - || elixir_do_section_call(node, code, ancestors).is_some_and(|call| { - crate::lang_helpers::elixir::elixir_call_keyword(&call, code) == Some("for") - }) -} - -/// Returns `true` when `node` is its parent's first `stab_clause` — -/// the head clause of a construct [`elixir_clauses_skip_head`] names. -/// -/// The parent is the `anonymous_function` (`fn stab_clause+ end`), the -/// `do_block` (`do stab_clause+ end`) or the keyword form's -/// parenthesised `block`; in each the first child whose kind is -/// `stab_clause` (skipping the `fn` / `do` / `(` token) is the head. -fn elixir_is_first_clause<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - ancestors - .parent(node) - .and_then(|parent| { - parent - .children() - .find(|child| child.kind_id() == Elixir::StabClause as u16) - }) - .is_some_and(|first| first.id() == node.id()) -} - -/// Whether `node` (a `stab_clause`) is one decision of the construct -/// that owns it: every clause of a `case` / `cond` / `with`'s `else` / -/// `receive` / `try` handler and every clause after a multi-clause -/// `fn`'s first, except the construct's free default clause -/// ([`elixir_is_default_clause`]). -/// -/// An anonymous fn's head clause is the closure's definition, not a -/// dispatch: the closure opens its own function space whose base path -/// already covers it, so counting it too would score a trivial -/// `fn x -> x end` as a decision (#776). A `for … reduce:`'s first -/// accumulator clause is skipped for the same reason: the `for` already -/// pays its one decision, and a single `acc -> …` clause always matches -/// (#1535). -/// -/// The parent is allowlisted because the grammar puts a `stab_clause` -/// in one more position: a parenthesised `block`, which is how a -/// typespec spells a function type (`@spec f((any -> any)) :: list`). -/// That is type syntax and branches on nothing, the same reason -/// [`elixir_when_is_guard`] excludes a typespec's `when` (#1531). The -/// keyword form of a clause construct (`case(x, do: (1 -> :a; …))`) -/// holds its clauses in the same parenthesised `block`, so a `block` -/// counts when it is the value of a section pair -/// ([`elixir_is_section_pair`]). -/// -/// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode`, which -/// both score a clause construct per clause and must agree on which -/// clauses those are (grammar-dispatch §7, #1531). -pub(crate) fn elixir_clause_is_decision<'a>( - node: &Node<'a>, - code: &'a [u8], - ancestors: Ancestors<'a, '_>, -) -> bool { - use Elixir as E; - - let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); - let in_section = up - .next() - .is_some_and(|parent| match parent.kind_id().into() { - E::DoBlock - | E::ElseBlock - | E::AfterBlock - | E::RescueBlock - | E::CatchBlock - | E::AnonymousFunction => true, - E::Block => up - .next() - .is_some_and(|pair| elixir_is_section_pair(&pair, code, SECTION_KEYS)), - _ => false, - }); - if !in_section { - return false; - } - let skips_head = elixir_clauses_skip_head(node, code, ancestors); - if skips_head && elixir_is_first_clause(node, ancestors) { - return false; - } - !elixir_is_default_clause(node, code, ancestors, skips_head) -} - -/// Whether a `with` Call's `else` clauses dispatch: whether at least one -/// of them is a decision rather than the free bare `_ ->` default. -/// -/// Modified cyclomatic collapses a clause construct's arms into one -/// decision, which for `case` / `cond` / `try` is the container paying -/// once. A `with`'s decisions are its `<-` clauses, each paid on its own -/// in both tiers as a Rust `if let` is; its `else` is the dispatch over -/// the failed value, which collapses into one decision exactly when it -/// holds one — so `else _ -> 0`, which picks nothing, stays free in -/// modified as it is in standard, like Rust's `if let … else` (#1535). -/// Spelled in either form: the block form's `else_block`, or the -/// keyword form's `else:` pair whose value is a parenthesised `block`. -/// -/// In a `with`'s `else` the default-clause rule reduces to the bare -/// `_`: `true ->` is free only under a `cond`, and the head skip only -/// under a `fn` or `reduce:`, so [`elixir_clause_is_decision`] agrees -/// clause by clause with the test below. -pub(crate) fn elixir_with_else_dispatches(call: &Node, code: &[u8]) -> bool { - use Elixir as E; - - call.children().any(|child| { - let section = match child.kind_id().into() { - E::DoBlock => child - .children() - .find(|c| c.kind_id() == E::ElseBlock as u16), - _ if child.kind() == "arguments" => child - .children() - .filter(|c| c.kind_id() == E::Keywords as u16) - .flat_map(|keywords| keywords.children()) - .find(|pair| elixir_is_section_pair(pair, code, &["else:"])) - .and_then(|pair| pair.child_by_field_name("value")), - _ => None, - }; - section.is_some_and(|section| { - section.children().any(|clause| { - clause.kind_id() == E::StabClause as u16 - && !elixir_sole_unguarded_pattern(&clause) - .is_some_and(|pattern| elixir_is_bare_wildcard(&pattern, code)) - }) - }) - }) -} - -/// The keyword spellings of the clause sections a `do … end` block -/// spells as `do_block` / `else_block` / `after_block` / `rescue_block` -/// / `catch_block`. -const SECTION_KEYS: &[&str] = &["do:", "else:", "after:", "rescue:", "catch:"]; - -/// Whether `pair` is a keyword-form section (`do: (…)`) whose key is one -/// of `keys`. The key token's text carries its colon and the whitespace -/// after it, so it is trimmed before comparing. -fn elixir_is_section_pair(pair: &Node, code: &[u8], keys: &[&str]) -> bool { - pair.kind_id() == Elixir::Pair as u16 - && pair - .child_by_field_name("key") - .and_then(|key| key.utf8_text(code)) - .is_some_and(|key| keys.contains(&key.trim_end())) -} - -/// The `Call` whose `do` section holds `node` (a `stab_clause`): the -/// block form's `do_block` (whose parent is the call) or the keyword -/// form's `do: (…)` block (`block` → `pair` → `keywords` → `arguments` -/// → call). The caller asks the Call's keyword (`cond`, `for`). -fn elixir_do_section_call<'a>( - node: &Node<'a>, - code: &'a [u8], - ancestors: Ancestors<'a, '_>, -) -> Option> { - use Elixir as E; - - let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); - match up.next().map(|parent| parent.kind_id().into()) { - Some(E::DoBlock) => up.next(), - Some(E::Block) => up - .next() - .filter(|pair| elixir_is_section_pair(pair, code, &["do:"])) - .and_then(|_| up.nth(2)), - _ => None, - } -} - // A `visibility_modifier` node counts as public unless it has a direct // `Zelf` child — the structural signature of `pub(self)` / `pub(in self)`, // which restrict visibility to the current module (semantically private, From 6597164bad48a31566ac61d000a48445185b3527 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:59:01 -0700 Subject: [PATCH 18/48] fix(cyclomatic/rust): count let-else as a decision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `let PAT = e else { … };` binds or diverges, the same decision its `if let` and `match` twins pay, but no Rust cyclomatic arm matched it, so it scored nothing and broke the `abc.conditions == cyclomatic - 1` parity ABC's `else` slot already held. Count a `let_declaration` that carries an `alternative` block, +1 in standard and modified. Keying on the field rather than the `else` token leaves `if … else` alone. Cognitive's existing 1 comes from the shared `else` token arm, not a let-else rule, and takes no nesting; that is filed as #1548. The serde corpus moves in one file (three let-elses). The self-scan baseline is unchanged: 151 functions gain +175 in total, and none crosses the soft or hard cyclomatic limit. Fixes #1542 --- big-code-analysis-book/src/metrics.md | 5 +- src/metrics/cognitive/rust.rs | 4 +- src/metrics/cyclomatic.rs | 93 +++++++++++++++++++++ src/metrics/cyclomatic/rust.rs | 8 ++ tests/repositories/big-code-analysis-output | 2 +- 5 files changed, 109 insertions(+), 3 deletions(-) diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 99b24c4b..53cc9eb0 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -504,7 +504,10 @@ meet: each `pattern <- expr` clause is a separate test, scored once in both variants, as Rust's `if let` and `?` are. The clauses of its `else` are alternatives, so modified scores them once, and only when one of them is a decision: `else _ -> 0` selects nothing, like a plain -`else`, and is free in both variants (#1535). +`else`, and is free in both variants (#1535). Rust's `let`-`else` +(`let Ok(a) = g(x) else { return 0 };`) is one such test too. It binds +or diverges, so it scores one in both variants, as its `if let` twin +does. Before #1542 it scored nothing. ### Counting Rust's `?` operator diff --git a/src/metrics/cognitive/rust.rs b/src/metrics/cognitive/rust.rs index 8495a81d..1f97cac7 100644 --- a/src/metrics/cognitive/rust.rs +++ b/src/metrics/cognitive/rust.rs @@ -34,7 +34,9 @@ impl Cognitive for RustCode { } // `Else` here is the `else` keyword token, which the grammar // also emits for the `else` of an `else if` — so this arm - // covers both. + // covers both. A `let`-`else` reaches it too, as a flat +1 + // with no nesting; that is a side effect of the shared token, + // not a let-else rule (#1548). Else => { increment_by_one(stats); } diff --git a/src/metrics/cyclomatic.rs b/src/metrics/cyclomatic.rs index 8111aa1a..59012d3c 100644 --- a/src/metrics/cyclomatic.rs +++ b/src/metrics/cyclomatic.rs @@ -7373,6 +7373,99 @@ end ); } + // A `let`-`else` binds or diverges: one decision (#1542). Each row is + // one `fn`, read as (cyclomatic, modified, abc.conditions). + // + // Expected values come from the twins, not the fix (measured, + // `--no-config`): `il` (`if let`) and `m` (`match` with a bare `_`) + // both score 2/2/1, so `le` must as well. `ctrl` is the same body + // without the `else`, at 1/1/0. `le2` pays once per `let`-`else`, 3/3/2. + // `ie` pins that the shared `else` token of an `if … else` still pays + // nothing extra (2/2, with ABC's slot for `x` and the `else`). `lp` + // nests one in a `for`: 1 + `for` + `let`-`else` = 3, where ABC counts + // only the `else`, because it does not score loops. `cl`'s closure + // opens its own space, which pays the decision itself. + #[cfg(feature = "rust")] + #[test] + fn rust_let_else_is_one_decision_1542() { + let src = "fn le(x: i32) -> i32 { + let Ok(a) = g(x) else { return 0 }; + a +} +fn ctrl(x: i32) -> i32 { + let a = g(x); + a +} +fn il(x: i32) -> i32 { + if let Ok(a) = g(x) { return a; } + 0 +} +fn m(x: i32) -> i32 { + match g(x) { Ok(a) => a, _ => 0 } +} +fn le2(x: i32) -> i32 { + let Ok(a) = g(x) else { return 0 }; + let Some(b) = h(a) else { return 1 }; + b +} +fn ie(x: bool) -> i32 { + if x { 1 } else { 0 } +} +fn lp(xs: &[Option]) { + for y in xs { + let Some(a) = y else { continue }; + k(a); + } +} +fn cl() { + let c = |y: Option| { + let Some(a) = y else { return 0 }; + a + }; + k(c); +} +"; + let expected: &[(&str, u64, u64, u64)] = &[ + ("le", 2, 2, 1), + ("ctrl", 1, 1, 0), + ("il", 2, 2, 1), + ("m", 2, 2, 1), + ("le2", 3, 3, 2), + ("ie", 2, 2, 2), + ("lp", 3, 3, 1), + ("cl", 1, 1, 0), + ]; + check_func_space_only::( + src, + "foo.rs", + &[Metric::Cyclomatic, Metric::Abc], + |space| { + assert_eq!(space.spaces.len(), expected.len(), "fixture members"); + for &(name, cyclomatic, modified, conditions) in expected { + let f = child_space(&space, name); + let measured = ( + f.metrics.cyclomatic.cyclomatic(), + f.metrics.cyclomatic.cyclomatic_modified(), + f.metrics.abc.conditions(), + ); + assert_eq!(measured, (cyclomatic, modified, conditions), "{name}"); + } + let closure = child_space(&space, "cl"); + assert_eq!(closure.spaces.len(), 1, "cl's closure space"); + let closure = &closure.spaces[0].metrics; + assert_eq!( + ( + closure.cyclomatic.cyclomatic(), + closure.cyclomatic.cyclomatic_modified(), + closure.abc.conditions(), + ), + (2, 2, 1), + "cl's closure" + ); + }, + ); + } + /// Regression #107: empty case…esac has no arms, so standard adds 0 and /// modified adds 1 (the container). #[cfg(feature = "bash")] diff --git a/src/metrics/cyclomatic/rust.rs b/src/metrics/cyclomatic/rust.rs index 03a7666f..e5eedbd1 100644 --- a/src/metrics/cyclomatic/rust.rs +++ b/src/metrics/cyclomatic/rust.rs @@ -80,6 +80,14 @@ fn rust_cyclomatic_increment(node: &Node<'_>, stats: &mut Stats, count_try: bool stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } + // `let PAT = e else { … };` either binds or diverges: one + // decision, as its `if let` twin pays. Keyed on the + // `alternative` field rather than the shared `else` token, which + // `if … else` also emits. + LetDeclaration if node.child_by_field_name("alternative").is_some() => { + stats.cyclomatic += 1.; + stats.cyclomatic_modified += 1.; + } // Both standard and modified. If | For | While | Loop | AMPAMP | PIPEPIPE => { stats.cyclomatic += 1.; diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index 256811a8..dd519f90 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit 256811a850866f321bca53d387722829606dbe4d +Subproject commit dd519f90c70d392848dc00677e5ebc1fd93d03fd From 11a61d4d312eda12f09957bcc6ecfd2c4527d92c Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:26:01 -0700 Subject: [PATCH 19/48] fix(getter/perl): bill -bareword keys as operands tree-sitter-perl 1.1.2 lexes the `-X` of an auto-quoted `-bareword` key as a file test. `$h{-foo}` and `(-text => 1)` billed the operand `oo` / `ext` plus an ABC branch for "calling" it, `(-x => 1)` billed the whole `-x => 1` span, and `-not` / `-and` keys billed a `-` operator. The Perl getter now bills a glued `-bareword` file test in key position (whole hash subscript key, or left of `=>`) as one operand spelling the key, drops the inner word, and drops the `-` of a `-and` / `-not` key while spelling that key with its sign. The ABC walker no longer counts the inner word as a call. Real file tests, and `-bareword` outside a key, keep their reading. The subscript form of an exact file-test key (`$h{-x}`) drops the token in error recovery and stays unbilled. Fixes #1545 --- .bca-baseline.toml | 2 +- big-code-analysis-ast/src/getter/perl.rs | 118 ++++++++-- .../src/lang_helpers/perl.rs | 129 ++++++++++- src/metrics/abc/perl.rs | 9 +- src/metrics/perl_keyword_key_tests.rs | 201 +++++++++++++++++- 5 files changed, 429 insertions(+), 30 deletions(-) diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 0701b103..f90b9d7a 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -243,7 +243,7 @@ value = 51602.8394982239 path = "big-code-analysis-ast/src/getter/perl.rs" qualified = "PerlCode::get_op_type" metric = "halstead.effort" -value = 120979.88227622502 +value = 126401.3661092431 [[entry]] path = "big-code-analysis-ast/src/getter/php.rs" diff --git a/big-code-analysis-ast/src/getter/perl.rs b/big-code-analysis-ast/src/getter/perl.rs index 9691d5ff..acbc949c 100644 --- a/big-code-analysis-ast/src/getter/perl.rs +++ b/big-code-analysis-ast/src/getter/perl.rs @@ -2,7 +2,10 @@ #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] use super::*; -use crate::lang_helpers::perl::{perl_and_is_operator, perl_not_is_key}; +use crate::lang_helpers::perl::{ + perl_and_is_operator, perl_dash_signs_keyword_key, perl_file_test_is_dash_key, + perl_is_dash_key_word, perl_not_is_key, +}; impl Getter for PerlCode { fn get_space_kind(node: &Node) -> SpaceKind { @@ -18,6 +21,9 @@ impl Getter for PerlCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Perl as P; + if let Some(role) = auto_quoted_key_role(node, ancestors) { + return role; + } match node.kind_id().into() { // Regex delimiter punctuation. The bare match form is the // only one of Perl's five regex literals that spells its @@ -38,27 +44,6 @@ impl Getter for PerlCode { { TokenRole::Unknown } - // FIXME(#1539 upstream): an auto-quoted `and` key (`$h{and}`, - // `(and => 1)`) lexes as the operator keyword. It is a - // string key, so it bills what `$h{or}` bills: the - // `identifier` operand the grammar gives every other - // bareword key. - P::And - if !ancestors - .parent(node) - .is_some_and(|parent| perl_and_is_operator(node, &parent)) => - { - TokenRole::Operand - } - // FIXME(#1541 upstream): the same for an auto-quoted `not` - // key (`$h{not}`, `(not => 1)`); see `perl_not_is_key`. - P::Not - if ancestors - .parent(node) - .is_some_and(|parent| perl_not_is_key(node, &parent)) => - { - TokenRole::Operand - } // Control-flow and declaration keywords. `Perl::Sub` is the // `sub` keyword (token id 16); `Perl::SUB` is the `__SUB__` // literal (token id 7) — that one is an operand, not an @@ -252,6 +237,35 @@ impl Getter for PerlCode { } } + /// Spells a `-bareword` key whole (#1545). A key that is exactly a + /// file test (`(-x => 1)`) has swallowed the `=>` and the value into + /// its node, so the key is the sign and the word after it; a + /// `-and` / `-not` key gets back the sign its `-` token stopped + /// billing. Only a key reaches here as an operand, so the byte + /// before `and` / `not` is that sign whenever it is a `-`. + fn get_operand_id<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + ) -> &'a [u8] { + let (start, end) = (node.start_byte(), node.end_byte()); + match node.kind_id().into() { + Perl::FileHandleOperator => { + let word = code[start + 1..end] + .iter() + .take_while(|byte| byte.is_ascii_alphanumeric() || **byte == b'_') + .count(); + &code[start..=start + word] + } + Perl::And | Perl::Not + if start.checked_sub(1).is_some_and(|sign| code[sign] == b'-') => + { + &code[start - 1..end] + } + _ => &code[start..end], + } + } + /// Folds paired delimiters to one glyph, and names the two /// pattern *operations* after their source spelling. /// @@ -276,3 +290,63 @@ impl Getter for PerlCode { } } } + +/// The role of a token tree-sitter-perl 1.1.2 misparses inside an +/// auto-quoted hash key, or `None` for any other token. Perl quotes a +/// bareword before `=>` and alone in a hash subscript, keywords and a +/// leading `-` included, and the grammar reads some of those keys as +/// operators. Every arm is an upstream workaround; drop each with its +/// issue once the pin carries the fix. +fn auto_quoted_key_role<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> Option { + use Perl as P; + + let role = match node.kind_id().into() { + // FIXME(#1539 upstream): an auto-quoted `and` key (`$h{and}`, + // `(and => 1)`) lexes as the operator keyword. It is a + // string key, so it bills what `$h{or}` bills: the + // `identifier` operand the grammar gives every other + // bareword key. + P::And + if !ancestors + .parent(node) + .is_some_and(|parent| perl_and_is_operator(node, &parent)) => + { + TokenRole::Operand + } + // FIXME(#1541 upstream): the same for an auto-quoted `not` + // key (`$h{not}`, `(not => 1)`); see `perl_not_is_key`. + P::Not + if ancestors + .parent(node) + .is_some_and(|parent| perl_not_is_key(node, &parent)) => + { + TokenRole::Operand + } + // FIXME(#1545 upstream): a `-bareword` key (`$h{-foo}`, + // `(-text => 1)`) lexes as a file test on the rest of the + // word. It is one string key, so the test node bills it as + // one operand spelling the whole key, and the word inside + // bills nothing. A `-and` / `-not` key splits instead into + // a `-` and the keyword, which the arms above bill as the + // key; the `-` is part of it and bills nothing. See + // `perl_file_test_is_dash_key`. + P::FileHandleOperator if perl_file_test_is_dash_key(node, ancestors) => TokenRole::Operand, + P::Identifier + if ancestors + .iter(node) + .next() + .is_some_and(|(word, above)| perl_is_dash_key_word(&word, above)) => + { + TokenRole::Unknown + } + P::DASH + if ancestors + .parent(node) + .is_some_and(|parent| perl_dash_signs_keyword_key(node, &parent)) => + { + TokenRole::Unknown + } + _ => return None, + }; + Some(role) +} diff --git a/big-code-analysis-ast/src/lang_helpers/perl.rs b/big-code-analysis-ast/src/lang_helpers/perl.rs index 029c88ed..1e192f51 100644 --- a/big-code-analysis-ast/src/lang_helpers/perl.rs +++ b/big-code-analysis-ast/src/lang_helpers/perl.rs @@ -1,8 +1,8 @@ -//! Perl: telling the `and` and `not` operators from an auto-quoted -//! `and` or `not` key. +//! Perl: telling the `and` and `not` operators, and the file tests, +//! from an auto-quoted key that the grammar lexes as one of them. use crate::Perl; -use crate::node::Node; +use crate::node::{Ancestors, Node}; /// Whether `and`, a Perl `and` token whose parent is `parent`, is the /// low-precedence logical operator. @@ -78,6 +78,129 @@ pub fn perl_not_is_key(not: &Node, parent: &Node) -> bool { .is_some_and(|next| next.is_error() && next.is_child(Perl::FatComma as u16)) } +/// Whether `test`, a Perl `file_handle_operator` whose ancestor chain is +/// `ancestors`, is an auto-quoted `-bareword` key rather than a file +/// test. +/// +/// FIXME(#1545 upstream): tree-sitter-perl 1.1.2 lexes the leading `-X` +/// of a dash-prefixed bareword key as a file test. Perl auto-quotes +/// `-foo` before `=>` and alone in a hash subscript — the Tk, CGI and +/// Getopt option-hash idiom — so `$h{-foo}` and `(-text => 1)` hold the +/// strings `"-foo"` and `"-text"`, and no operator. The grammar emits +/// two shapes: +/// +/// - `-` and a letter glued to more word characters (`-foo`) becomes a +/// file test on the bareword `oo`. Perl never reads that as a file +/// test, since one needs a non-word character after its letter, so +/// only the position decides: the test is the whole `key` of a hash +/// subscript, or the left of a `=>`. Elsewhere `-foo` is a negated +/// call or a string, and stays as the grammar reads it. +/// - a key that *is* a file test (`(-x => 1)`) swallows the `=>` and +/// the value, wrapping the `=>` in an `ERROR`. A real file test +/// cannot take a `=>` as its operand, so the `ERROR` decides. +/// +/// The subscript form of the second shape (`$h{-x}`) recovers into an +/// `ERROR` that drops the `-x` token altogether, so nothing is left to +/// bill. Remove this, and every call site, once the pin carries the +/// upstream fix. +#[must_use] +pub(crate) fn perl_file_test_is_dash_key<'a>( + test: &Node<'a>, + ancestors: Ancestors<'a, '_>, +) -> bool { + test.children() + // As in `perl_not_is_key`: the `ERROR` is an extra. + .find(|child| child.is_error() || !child.as_tree_sitter().is_extra()) + .is_some_and(|operand| { + if operand.is_error() { + operand.is_child(Perl::FatComma as u16) + } else { + is_glued_word(&operand, test) && is_key_position(test, ancestors) + } + }) +} + +/// Whether `node` is the word of a `-bareword` key that the enclosing +/// file test bills whole, so it bills nothing of its own. `ancestors` +/// is `node`'s chain. +#[must_use] +pub fn perl_is_dash_key_word<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + ancestors.iter(node).next().is_some_and(|(test, above)| { + test.kind_id() == Perl::FileHandleOperator as u16 + && is_glued_word(node, &test) + && perl_file_test_is_dash_key(&test, above) + }) +} + +/// Whether `dash`, a Perl `-` token whose parent is `parent`, is the +/// sign of an auto-quoted `-and` or `-not` key (`$h{-not}`, +/// `(-and => 1)`), which the grammar splits into the `-` and the keyword +/// token. [`perl_and_is_operator`] and [`perl_not_is_key`] decide +/// whether the keyword is a key, and a key's sign is part of the string +/// whether or not a space separates the two. +#[must_use] +pub(crate) fn perl_dash_signs_keyword_key(dash: &Node, parent: &Node) -> bool { + // `$h{-not}` keeps the keyword beside the `-`; `(-not => 1)` nests + // it as the first child of the `-`'s operand. + next_non_extra(parent, dash).is_some_and(|next| { + keyword_is_key(&next, parent) + || first_non_extra(&next).is_some_and(|keyword| keyword_is_key(&keyword, &next)) + }) +} + +fn keyword_is_key(token: &Node, parent: &Node) -> bool { + match token.kind_id().into() { + Perl::And => !perl_and_is_operator(token, parent), + Perl::Not => perl_not_is_key(token, parent), + _ => false, + } +} + +// Whether `word` is the bareword the grammar split off a `-bareword` +// key: the `-X` file-test token is two bytes, so a word glued to it +// starts right after them. +fn is_glued_word(word: &Node, test: &Node) -> bool { + word.kind_id() == Perl::CallExpressionWithBareword as u16 + && word.start_byte() == test.start_byte() + 2 +} + +// Whether the `unary_expression` around `test` is the whole key of a +// hash subscript, which the grammar wraps in a `binary_expression` with +// no other child but comments, or is followed by a `=>`. +fn is_key_position<'a>(test: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let mut up = ancestors.iter(test).map(|(ancestor, _)| ancestor); + let mut key = up.next(); + let mut parent = up.next(); + if parent + .is_some_and(|p| p.kind_id() == Perl::BinaryExpression as u16 && non_extra_count(&p) == 1) + { + key = parent; + parent = up.next(); + } + key.zip(parent).is_some_and(|(key, parent)| { + matches!( + parent.kind_id().into(), + Perl::HashAccessVariableSimple | Perl::HashAccessVariable + ) || next_non_extra(&parent, &key) + .is_some_and(|next| next.kind_id() == Perl::FatComma as u16) + }) +} + +fn non_extra_count(node: &Node) -> usize { + node.children() + .filter(|child| !child.as_tree_sitter().is_extra()) + .count() +} + +// The sibling after `child` in `parent`, comments aside. +fn next_non_extra<'a>(parent: &Node<'a>, child: &Node) -> Option> { + parent + .children() + .filter(|sibling| !sibling.as_tree_sitter().is_extra()) + .skip_while(|sibling| sibling.id() != child.id()) + .nth(1) +} + fn first_non_extra<'a>(node: &Node<'a>) -> Option> { node.children() .find(|child| !child.as_tree_sitter().is_extra()) diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index 6f289874..3f514094 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -13,7 +13,7 @@ use super::{ Abc, Stats, count_boolean_slot, count_each_operand, count_negated_operand, is_operand, last_operand, wrapped_operand, }; -use crate::lang_helpers::perl::{perl_and_is_operator, perl_not_is_key}; +use crate::lang_helpers::perl::{perl_and_is_operator, perl_is_dash_key_word, perl_not_is_key}; use crate::*; // Fitzpatrick's ABC rules adapted for Perl. @@ -456,7 +456,10 @@ impl Abc for PerlCode { // when this node is the outermost dispatch site. When the // bareword sits inside one of the wrappers above, the // outer node has already been counted and this child - // would double the branch tally. + // would double the branch tally. Nor is the word of a + // `-bareword` key a call (FIXME(#1545 upstream)): the + // grammar splits `$h{-foo}` into a file test on a bareword + // `oo`, and the key is a string. P::CallExpressionWithBareword if !ancestors.parent(node).is_some_and(|p| { matches!( @@ -467,7 +470,7 @@ impl Abc for PerlCode { | P::CallExpressionWithVariable | P::CallExpressionRecursive ) - }) => + }) && !perl_is_dash_key_word(node, ancestors) => { stats.branches += 1.; } diff --git a/src/metrics/perl_keyword_key_tests.rs b/src/metrics/perl_keyword_key_tests.rs index ca505cc0..4f1f9f78 100644 --- a/src/metrics/perl_keyword_key_tests.rs +++ b/src/metrics/perl_keyword_key_tests.rs @@ -1,5 +1,7 @@ //! An auto-quoted `and` or `not` key scores like any other key, and the //! `and` and `not` operators still score as operators (#1539, #1541). +//! The same holds for a `-bareword` key and the file tests the grammar +//! mistakes it for (#1545). //! //! Perl quotes a bareword before `=>` and alone in a hash subscript, so //! `$h{and}` and `(not => 1)` hold a string key and no operator. @@ -32,7 +34,15 @@ type Halstead = [u64; 4]; #[cfg(feature = "perl")] fn measure(body: &str) -> (Decisions, Halstead) { - let out = Cell::new(([0; 3], [0; 4])); + let ([cyclomatic, _, conditions, cognitive], halstead) = measure_with_branches(body); + ([cyclomatic, conditions, cognitive], halstead) +} + +/// `[cyclomatic, abc branches, abc conditions, cognitive]` sums of the +/// root space, and its Halstead counts. +#[cfg(feature = "perl")] +fn measure_with_branches(body: &str) -> ([u64; 4], Halstead) { + let out = Cell::new(([0; 4], [0; 4])); check_func_space_only::( &format!("sub f {{ {body} }}\n"), "foo.pl", @@ -47,6 +57,7 @@ fn measure(body: &str) -> (Decisions, Halstead) { out.set(( [ m.cyclomatic.cyclomatic_sum(), + m.abc.branches_sum(), m.abc.conditions_sum(), m.cognitive.cognitive_sum(), ], @@ -300,3 +311,191 @@ fn perl_not_operator_still_scores_like_bang() { ); } } + +/// `(quoted_twin, dash_key, [cyclomatic, branches, conditions, +/// cognitive], halstead)` rows for +/// `perl_dash_key_scores_like_its_quoted_twin`. +#[cfg(feature = "perl")] +const DASH_KEY_ROWS: [(&str, &str, [u64; 4], Halstead); 18] = [ + ( + "my %h; return $h{'-foo'};", + "my %h; return $h{-foo};", + [2, 0, 0, 0], + [6, 8, 4, 4], + ), + // A comment in the subscript. Its `#` bills as an operator in both + // spellings (#1549). + ( + "my %h; return $h{'-foo' # why\n};", + "my %h; return $h{-foo # why\n};", + [2, 0, 0, 0], + [7, 9, 4, 4], + ), + ( + "my $r; return $r->{'-foo'};", + "my $r; return $r->{-foo};", + [2, 0, 0, 0], + [7, 10, 3, 4], + ), + ( + "my %h; return @h{'-foo'};", + "my %h; return @h{-foo};", + [2, 0, 0, 0], + [5, 7, 4, 4], + ), + ( + "my %h; if ($h{'-foo'}) { return 1; } return 0;", + "my %h; if ($h{-foo}) { return 1; } return 0;", + [3, 0, 1, 1], + [8, 13, 6, 6], + ), + ( + "my %h = ('-foo' => 1); return 1;", + "my %h = (-foo => 1); return 1;", + [2, 0, 0, 0], + [8, 9, 4, 5], + ), + ( + "my $r = { '-foo' => 1 }; return $r;", + "my $r = { -foo => 1 }; return $r;", + [2, 0, 0, 0], + [8, 11, 4, 5], + ), + ( + "f('-text' => 1);", + "f(-text => 1);", + [2, 1, 0, 0], + [5, 5, 3, 4], + ), + ( + "f('-text' => 1, ext => 2);", + "f(-text => 1, ext => 2);", + [2, 2, 0, 0], + [6, 7, 5, 6], + ), + ( + "my %h = ('-x' => 1); return 1;", + "my %h = (-x => 1); return 1;", + [2, 0, 0, 0], + [8, 9, 4, 5], + ), + ( + "my %h = ('-x' => 1); my %g = ('-x' => 2);", + "my %h = (-x => 1); my %g = (-x => 2);", + [2, 0, 0, 0], + [7, 12, 6, 7], + ), + // The swallowed value is still billed, call and all. + ( + "f('-x' => foo);", + "f(-x => foo);", + [2, 2, 0, 0], + [5, 5, 3, 4], + ), + ( + "my %h; return $h{'-not'};", + "my %h; return $h{-not};", + [2, 0, 0, 0], + [6, 8, 4, 4], + ), + ( + "my $r; return $r->{'-and'};", + "my $r; return $r->{-and};", + [2, 0, 0, 0], + [7, 10, 3, 4], + ), + ( + "my %h = ('-and' => 1); return 1;", + "my %h = (-and => 1); return 1;", + [2, 0, 0, 0], + [8, 9, 4, 5], + ), + ( + "my $r = { '-not' => 1 }; return $r;", + "my $r = { -not => 1 }; return $r;", + [2, 0, 0, 0], + [8, 11, 4, 5], + ), + ( + "f('-not' => 1, not => 2);", + "f(-not => 1, not => 2);", + [2, 1, 0, 0], + [6, 7, 5, 6], + ), + ( + "f('-and' => 1, and => 2);", + "f(-and => 1, and => 2);", + [2, 1, 0, 0], + [6, 7, 5, 6], + ), +]; + +/// Every valid spelling of a `-bareword` key scores exactly as its +/// quoted twin does, in all four metrics (#1545). tree-sitter-perl +/// reads `-foo` as the file test `-f` on a bareword `oo`, `-x => 1` as +/// the file test `-x` swallowing the `=>`, and `-not` as a `-` applied +/// to the keyword. Before the fix each `-foo` billed the operand `oo` +/// and an ABC branch for calling it, and each `-not` / `-and` billed a +/// `-` operator. +/// +/// The rows pairing a key with a second key equal to the misparsed +/// word (`ext`, `not`) or to itself (`-x` twice) pin the operand's +/// spelling, which the counts alone cannot see: billed as `ext`, `not` +/// or `-x => 1`, the two keys share or split one vocabulary entry +/// where their twins do not. +#[cfg(feature = "perl")] +#[test] +fn perl_dash_key_scores_like_its_quoted_twin() { + for (twin, key, decisions, halstead) in DASH_KEY_ROWS { + assert_eq!(twin.replace('\'', ""), key, "`{key}` drifted from its twin"); + let want = measure_with_branches(twin); + assert_eq!( + want, + (decisions, halstead), + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure_with_branches(key), + want, + "`{key}` must score like `{twin}`" + ); + } +} + +/// A `-X` that is not a key keeps the reading it had before #1545: a +/// real file test, alone, in a condition, on the `_` stat cache, +/// stacked, and as a subscript's whole expression, where only a word +/// glued to the letter makes a key; a `-bareword` outside key position, +/// including a subscript it is only part of; +/// and a `-` applied to a real `not`. The file test itself is billed as +/// nothing, since the grammar emits its `-X` as a hidden token. Every +/// value is `main`'s, measured before the fix. +#[cfg(feature = "perl")] +#[test] +fn perl_file_test_outside_a_key_keeps_its_reading() { + let rows: [(&str, [u64; 4], Halstead); 11] = [ + ("my $z = -f $x;", [2, 0, 0, 0], [6, 7, 3, 3]), + ( + "if (-e $x) { return 1; } return 0;", + [3, 0, 1, 1], + [7, 10, 4, 4], + ), + ("my $z = -d _;", [2, 1, 0, 0], [6, 6, 3, 3]), + ("my $z = -f -w $x;", [2, 0, 0, 0], [6, 7, 3, 3]), + ("my %h; return $h{-f $x};", [2, 0, 0, 0], [6, 9, 4, 4]), + ("my %h; return $h{-f foo};", [2, 1, 0, 0], [6, 8, 4, 4]), + ("return -foo;", [2, 1, 0, 0], [4, 4, 2, 2]), + // Inside a subscript, but not the whole key. + ("my %h; return $h{$a . -foo};", [2, 1, 0, 0], [7, 10, 5, 5]), + ("f(-foo, 1);", [2, 2, 0, 0], [5, 5, 3, 4]), + ("return -not $x;", [2, 0, 0, 0], [7, 7, 2, 2]), + ("return $a - not $x;", [2, 0, 0, 0], [7, 8, 3, 3]), + ]; + for (body, decisions, halstead) in rows { + assert_eq!( + measure_with_branches(body), + (decisions, halstead), + "`{body}` changed its reading" + ); + } +} From 38391f071273911b7fc8d97a2967f39518bdaf37 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:47:02 -0700 Subject: [PATCH 20/48] fix(getter/cpp): bill co_return and co_yield C++'s coroutine keywords were in neither Halstead arm, so `co_return` and `co_yield` billed nothing while `return` and, since #1296, `co_await` are operators. Both are now operators in Cpp and Mozcpp. Each is the keyword leaf of an unclassified statement node, so nothing bills one twice. NExits counted neither statement either. `co_return_statement` is a coroutine's `return`, and `co_yield_statement` suspends to the caller as the Python / JS `yield` NExits already counts, so both are exits. No DeepSpeech snapshot moves: the corpus has no coroutines. Fixes #1547 --- big-code-analysis-ast/src/getter/cpp.rs | 6 +++ big-code-analysis-ast/src/getter/mozcpp.rs | 3 ++ big-code-analysis-book/src/metrics.md | 8 +++- src/metrics/halstead.rs | 32 ++++++++++++++ src/metrics/nexits.rs | 49 +++++++++++++++++++++- 5 files changed, 94 insertions(+), 4 deletions(-) diff --git a/big-code-analysis-ast/src/getter/cpp.rs b/big-code-analysis-ast/src/getter/cpp.rs index 46b84e5f..be9368a7 100644 --- a/big-code-analysis-ast/src/getter/cpp.rs +++ b/big-code-analysis-ast/src/getter/cpp.rs @@ -159,6 +159,12 @@ impl Getter for CppCode { // `bca ops` both renders `[]` — the same choice Ruby makes for // its `def [](i)` method names. | LPARENRPAREN | LBRACKRBRACK | DQUOTEDQUOTE | CoAwait | Operator + // `co_return` and `co_yield` are the coroutine spellings of + // `return` and of handing a value back, so they bill as + // `return` does (#1547). Each is the keyword leaf of a + // `co_return_statement` / `co_yield_statement`, which no arm + // classifies, so nothing bills one twice. + | CoReturn | CoYield => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only diff --git a/big-code-analysis-ast/src/getter/mozcpp.rs b/big-code-analysis-ast/src/getter/mozcpp.rs index 1d274d4d..6d37967f 100644 --- a/big-code-analysis-ast/src/getter/mozcpp.rs +++ b/big-code-analysis-ast/src/getter/mozcpp.rs @@ -113,6 +113,9 @@ impl Getter for MozcppCode { // An overloaded operator's name, the `operator` keyword and // `co_await` — the derivation is on the Cpp twin (#1296). | LPARENRPAREN | LBRACKRBRACK | DQUOTEDQUOTE | CoAwait | Operator + // `co_return` / `co_yield`, billed as `return` is — the + // derivation is on the Cpp twin (#1547). + | CoReturn | CoYield => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 53cc9eb0..2e3c0331 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -730,7 +730,8 @@ tokens involved the same way they spell real operators: `co_await` in an expression. C++'s `[]` and `()` names are tokens of their own, so `bca ops` lists `operator[]`'s `[]` beside a subscript's `[]` as a second operator, as Ruby does for `def [](i)` - (#1296). + (#1296). C++'s coroutine keywords `co_return` and `co_yield` are + operators as `return` is (#1547). - **A named operator is one operand.** Elixir names an operator without applying it in a capture (`&==/2`, `&and/2`) or a remote call (`Kernel.||(a, b)`), and its grammar wraps the token in an @@ -1060,7 +1061,10 @@ ones that have accreted "just one more parameter" feature flags. **NExits** counts the number of distinct exit points from a function — every explicit `return`, every `throw` / `raise`, and -(in Rust) every `?` early-return. The implicit fall-through return at +(in Rust) every `?` early-return. A generator's `yield` counts in +Python, the JavaScript family, C# and PHP, because it hands control +back to the caller, and so does C++'s `co_yield`; C++'s `co_return` is +a `return` (#1547). The implicit fall-through return at the end of a function is **not** counted; only explicit exits are (see issue #243). diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index e03c1424..1e5e25f4 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -10061,6 +10061,38 @@ S s; int a = s[1]; auto t = co_await s; assert_cpp_operator_names::("mozcpp"); } + /// Asserts a coroutine's `co_yield` and `co_return` keywords bill as + /// operators under parser `T`, as `return` does (#1547). + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + #[track_caller] + fn assert_cpp_coroutine_keywords(label: &str) { + let source = "T gen() { co_yield 1; co_yield 2; co_return 3; }"; + // Operators (n1 = 5, N1 = 8): `()` 1, `{}` 1, `co_yield` 2, + // `co_return` 1, `;` 3. Operands (n2 = 5, N2 = 5): `T`, `gen`, + // `1`, `2`, `3`. Before #1547: (3, 5, 5, 5), both keywords + // billed nowhere. + assert_halstead_counts::(source, "foo.cpp", [5, 8, 5, 5], label); + let ops = ops_of::(source, "foo.cpp"); + for keyword in ["co_yield", "co_return"] { + assert!( + ops.operators.iter().any(|o| o == keyword), + "{label}: `{keyword}` must be an operator; operators were {:?}", + ops.operators + ); + } + } + + /// Regression for #1547: `co_return` and `co_yield` are Halstead + /// operators. One row per clone, so reverting either arm fails its + /// own row; Mozcpp owns no file extension, so this row is its only + /// coverage. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_coroutine_keywords_are_operators() { + assert_cpp_coroutine_keywords::("cpp"); + assert_cpp_coroutine_keywords::("mozcpp"); + } + /// Regression for #1361: a C++ `this` is a Halstead operand. /// /// `Cpp::This` / `Mozcpp::This` were in neither arm of diff --git a/src/metrics/nexits.rs b/src/metrics/nexits.rs index 3c355a78..347c1a49 100644 --- a/src/metrics/nexits.rs +++ b/src/metrics/nexits.rs @@ -181,8 +181,29 @@ impl_exit_match_kinds!( Tsx, [ReturnStatement, ThrowStatement, YieldExpression] ); -impl_exit_match_kinds!(CppCode, Cpp, [ReturnStatement, ThrowStatement]); -impl_exit_match_kinds!(MozcppCode, Mozcpp, [ReturnStatement, ThrowStatement]); +// A coroutine's `co_return` is its `return`, and `co_yield` suspends it +// back to the caller as a Python or JS `yield` does, so both count +// (#1547). +impl_exit_match_kinds!( + CppCode, + Cpp, + [ + ReturnStatement, + ThrowStatement, + CoReturnStatement, + CoYieldStatement + ] +); +impl_exit_match_kinds!( + MozcppCode, + Mozcpp, + [ + ReturnStatement, + ThrowStatement, + CoReturnStatement, + CoYieldStatement + ] +); // C has no exceptions: `return` is the only exit kind (no `throw`). impl_exit_match_kinds!(CCode, C, [ReturnStatement]); // Objective-C adds `@throw` on top of C's `return` (the `throw_statement` @@ -2900,6 +2921,30 @@ end", ); } + /// A coroutine with two `co_yield`s and a `co_return`, which count + /// as three exits, as Python's two `yield`s and a `return` do (#1547). + /// Before #1547 neither statement counted and the sum was 0. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const CPP_COROUTINE: &str = "T gen() { co_yield 1; co_yield 2; co_return; }"; + + // One test per clone, so reverting either impl fails its own test. + #[cfg(feature = "cpp")] + #[test] + fn cpp_co_return_and_co_yield_count_as_exits() { + check_metrics::(CPP_COROUTINE, "foo.cpp", |metric| { + assert_eq!(metric.nexits.nexits_sum(), 3); + }); + } + + // Mozcpp owns no file extension, so this is its only coverage. + #[cfg(feature = "mozcpp")] + #[test] + fn mozcpp_co_return_and_co_yield_count_as_exits() { + check_metrics::(CPP_COROUTINE, "foo.cpp", |metric| { + assert_eq!(metric.nexits.nexits_sum(), 3); + }); + } + #[cfg(feature = "python")] #[test] fn python_yield_counts_as_exit() { From c71cfbd38f8799b99a4ade5a0c0971ca76143906 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:05:19 -0700 Subject: [PATCH 21/48] fix(halstead): stop billing zero-width tokens Error recovery inserts zero-width tokens for what the source lacks: a MISSING `;`, `)` or identifier, and a few non-MISSING identifiers (C#, Elixir). Every getter that classifies the kind billed them, so an inserted identifier became an operand spelled "", in all 23 languages, and an inserted `;` a second `;` operator. Halstead counts written tokens, so compute_halstead now skips any zero-width node, which covers `bca metrics` and `bca ops` alike. 34 DeepSpeech C/C++ snapshots move in Halstead and mi only. The openfst dfs-visit.h `Destroy` (`dfs_state->~DfsState()`) drops the "" operand: n2 8 -> 7, N2 16 -> 15. Fixes #1546 --- big-code-analysis-book/src/metrics.md | 8 + src/metrics/halstead.rs | 170 ++++++++++++++++++++ tests/repositories/big-code-analysis-output | 2 +- 3 files changed, 179 insertions(+), 1 deletion(-) diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 2e3c0331..aed1399d 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -732,6 +732,14 @@ tokens involved the same way they spell real operators: subscript's `[]` as a second operator, as Ruby does for `def [](i)` (#1296). C++'s coroutine keywords `co_return` and `co_yield` are operators as `return` is (#1547). +- **Only written tokens count.** When tree-sitter cannot parse a + construct, its error recovery may insert zero-width tokens the source + does not contain: a `;`, a `)`, an identifier. None of them is billed + in either half. Before #1546 an inserted identifier became an operand + whose text is the empty string. One case is a C++ destructor called + through a template name (`p->~T()`), which tree-sitter-cpp cannot + parse. Another is a statement-position `typename` cast, which made + every inserted `;` a second `;` operator. - **A named operator is one operand.** Elixir names an operator without applying it in a capture (`&==/2`, `&and/2`) or a remote call (`Kernel.||(a, b)`), and its grammar wraps the token in an diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 1e5e25f4..d5e666e5 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -383,6 +383,17 @@ fn compute_halstead<'a, T: Getter + Checker>( ancestors: Ancestors<'a, '_>, halstead_maps: &mut HalsteadMaps<'a>, ) { + // Halstead counts the tokens written, and a zero-width node spells + // none, so it bills neither half (#1546). Error recovery is where + // they come from: every MISSING node tree-sitter inserts for a token + // the source lacks (a `;`, a `)`, an identifier) is zero-width, and + // so is the identifier C#'s and Elixir's recovery builds without + // flagging it MISSING. Checked here rather than per getter because + // every getter that classifies the kind would otherwise bill it, an + // identifier as the empty-string operand. + if node.start_byte() == node.end_byte() { + return; + } match T::get_op_type_with_code(node, code, ancestors) { TokenRole::Operator => { if T::is_primitive(node) { @@ -10082,6 +10093,165 @@ S s; int a = s[1]; auto t = co_await s; } } + /// Regression for #1546: a zero-width node is billed in neither + /// Halstead half. Error recovery inserts one for a token the source + /// lacks, usually flagged MISSING, and every getter that classifies + /// the kind billed it — an identifier as the empty-string operand. + /// One fixture per language, each a parse on which recovery inserts + /// an operand-classified zero-width node; before the fix every row + /// billed `""`. C#'s and Elixir's are not flagged MISSING, which is + /// why the guard keys on width: an `is_missing()` guard leaves those + /// two rows billing `""`. The rows pin no counts, since the recovery trees + /// are the grammars' to change (grammar-dispatch §6); + /// `cpp_missing_tokens_in_valid_code_are_not_billed` anchors two. + #[cfg(any( + feature = "bash", + feature = "c", + feature = "cpp", + feature = "csharp", + feature = "elixir", + feature = "go", + feature = "groovy", + feature = "irules", + feature = "java", + feature = "javascript", + feature = "kotlin", + feature = "lua", + feature = "mozcpp", + feature = "mozjs", + feature = "objc", + feature = "perl", + feature = "php", + feature = "python", + feature = "ruby", + feature = "rust", + feature = "tcl", + feature = "typescript", + ))] + #[test] + fn zero_width_nodes_bill_neither_halstead_half() { + fn has_zero_width_node(node: tree_sitter::Node) -> bool { + node.start_byte() == node.end_byte() || { + let mut cursor = node.walk(); + node.children(&mut cursor).any(has_zero_width_node) + } + } + + let cases: &[(LANG, &str, &str)] = &[ + #[cfg(feature = "bash")] + (LANG::Bash, "foo.sh", "echo a $( )\n"), + #[cfg(feature = "c")] + (LANG::C, "foo.c", "int x; void () { x = 1; }\n"), + #[cfg(feature = "cpp")] + (LANG::Cpp, "foo.cpp", "void f(S *s) {\n s->~D();\n}\n"), + #[cfg(feature = "csharp")] + (LANG::Csharp, "foo.cs", "class A { int = 1; }\n"), + #[cfg(feature = "elixir")] + (LANG::Elixir, "foo.ex", "x = a.\n"), + #[cfg(feature = "go")] + (LANG::Go, "foo.go", "package m\nfunc f() { a.() }\n"), + #[cfg(feature = "groovy")] + (LANG::Groovy, "foo.groovy", "def x = a.\n"), + #[cfg(feature = "irules")] + ( + LANG::Irules, + "foo.irule", + "when HTTP_REQUEST { set x [expr {1 + }] }\n", + ), + #[cfg(feature = "java")] + (LANG::Java, "Foo.java", "class A { int = 1; }\n"), + #[cfg(feature = "javascript")] + (LANG::Javascript, "foo.js", "let {a: } = b;\n"), + #[cfg(feature = "kotlin")] + (LANG::Kotlin, "foo.kt", "fun f(: Int) {}\n"), + #[cfg(feature = "lua")] + (LANG::Lua, "foo.lua", "x = a.\n"), + #[cfg(feature = "mozcpp")] + ( + LANG::Mozcpp, + "foo.cpp", + "void f(S *s) {\n s->~D();\n}\n", + ), + #[cfg(feature = "mozjs")] + (LANG::Mozjs, "foo.jsm", "let {a: } = b;\n"), + #[cfg(feature = "objc")] + (LANG::Objc, "foo.m", "@interface A\n- (void)f:;\n@end\n"), + #[cfg(feature = "perl")] + (LANG::Perl, "foo.pl", "sub { $x = ; }\n"), + #[cfg(feature = "php")] + (LANG::Php, "foo.php", ";\n"), + #[cfg(feature = "python")] + (LANG::Python, "foo.py", "for in x:\n pass\n"), + #[cfg(feature = "ruby")] + (LANG::Ruby, "foo.rb", "x = a.\n"), + #[cfg(feature = "rust")] + (LANG::Rust, "foo.rs", "fn f() { let x = a.; }\n"), + #[cfg(feature = "tcl")] + (LANG::Tcl, "foo.tcl", "puts [expr {1 + }]\n"), + #[cfg(feature = "typescript")] + (LANG::Typescript, "foo.ts", "let {a: } = b;\n"), + #[cfg(feature = "typescript")] + (LANG::Tsx, "foo.tsx", "let {a: } = b;\n"), + ]; + crate::test_support::assert_fixtures_present(cases); + + let mut billed = Vec::new(); + for (lang, file, source) in cases { + let ast = crate::test_support::parse_named(*lang, file, source); + assert!( + has_zero_width_node(ast.as_tree_sitter().root_node()), + "{lang:?}: `{source}` no longer recovers with a zero-width node; find another fixture" + ); + let ops = ast.ops().expect("ops walk must yield a top-level Ops"); + assert!( + !ops.operands.is_empty(), + "{lang:?}: `{source}` billed no operand at all" + ); + billed.extend(ops.operands.iter().any(String::is_empty).then_some(*lang)); + } + assert!( + billed.is_empty(), + "a zero-width node was billed as the empty operand in {billed:?}" + ); + } + + /// Asserts two valid C++ programs that tree-sitter-cpp 0.23.4 can only + /// parse by inserting MISSING tokens bill only the tokens written + /// (#1546), under parser `T`. Both come from the DeepSpeech corpus. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + #[track_caller] + fn assert_cpp_missing_tokens_unbilled(label: &str) { + // `openfst/dfs-visit.h:75`: an explicit destructor call through a + // template name. Recovery reads `s->~D`, + // `~`, `<`, `>`, `;`. Operands (n2 = 5, N2 = 6): `f`, `S`, `s` 2, + // `D`, `F`. Before #1546: (9, 10, 6, 7), the extra being `""`. + let destructor = "void f(S *s) {\n s->~D();\n}\n"; + assert_halstead_counts::(destructor, "foo.cpp", [9, 10, 5, 6], label); + let ops = ops_of::(destructor, "foo.cpp"); + assert_eq!(ops.operands, ["D", "F", "S", "f", "s"], "{label}"); + + // `kenlm/lm/search_trie.cc:286`: a `typename` functional cast + // used as a statement. Recovery inserts a MISSING `)` after + // `quant_` and a MISSING `;` after `order`. Operators (n1 = 8, + // N1 = 10): `void`, `()` 3, `{}`, `::`, `,`, `-`, `.`, the one + // written `;`. Operands (n2 = 7, N2 = 7): `f`, `MiddlePointer`, + // `quant_`, `order`, `2`, `Write`, `w`. Before #1546 the MISSING + // `;` was a second `;`: (8, 11, 7, 7). + let cast = "void f() {\n typename Quant::MiddlePointer(quant_, order - 2).Write(w);\n}\n"; + assert_halstead_counts::(cast, "foo.cpp", [8, 10, 7, 7], label); + } + + /// Regression for #1546 on the two C++ clones. Mozcpp owns no file + /// extension, so this row is its only coverage. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_missing_tokens_in_valid_code_are_not_billed() { + assert_cpp_missing_tokens_unbilled::("cpp"); + assert_cpp_missing_tokens_unbilled::("mozcpp"); + } + /// Regression for #1547: `co_return` and `co_yield` are Halstead /// operators. One row per clone, so reverting either arm fails its /// own row; Mozcpp owns no file extension, so this row is its only diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index dd519f90..92d630d1 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit dd519f90c70d392848dc00677e5ebc1fd93d03fd +Subproject commit 92d630d1c1537bc4edd72e95e2fe7a3f6886fac4 From bdf542fb5ab3c85027f6da04631abf656f9f40e1 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:25:08 -0700 Subject: [PATCH 22/48] fix(cognitive/rust): score let-else like if let MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `let PAT = e else { … };` paid cognitive only through the shared `else` token arm, a flat +1 with no nesting, so inside a `for` it scored one below its `if let` twin and fell further behind with each level. Score it as the `if let` it replaces: a structural increment plus the current nesting, keyed on the `alternative` field as cyclomatic is (#1542), with everything under it, the `else` block included, one level deeper. The `else` token arm now skips a token whose parent is a `let_declaration`, so nothing pays twice. Across the repository 50 spaces move, +62 in total. One function newly reaches the soft tier: `greedy_truck_factor` in src/vcs/bus_factor.rs goes 14 -> 15, the cognitive limit, and is baselined. No corpus snapshot moves; serde's three let-elses are at function top level, where both rules give 1. Fixes #1548 --- .bca-baseline.toml | 6 ++ big-code-analysis-book/src/metrics.md | 8 ++ src/metrics/cognitive.rs | 147 ++++++++++++++++++++++++++ src/metrics/cognitive/rust.rs | 16 ++- 4 files changed, 173 insertions(+), 4 deletions(-) diff --git a/.bca-baseline.toml b/.bca-baseline.toml index f90b9d7a..a3d8c0da 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -1187,6 +1187,12 @@ qualified = "compute" metric = "halstead.effort" value = 47694.803479864124 +[[entry]] +path = "src/vcs/bus_factor.rs" +qualified = "greedy_truck_factor" +metric = "cognitive" +value = 15.0 + [[entry]] path = "src/vcs/bus_factor.rs" qualified = "greedy_truck_factor" diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index aed1399d..27db2f47 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -395,6 +395,14 @@ Sonar ecosystem. levels up happened to be a closure. Python cannot express the shape at all: a `def` is a statement and a lambda body is a single expression. +- **Rust**'s `let`-`else` (`let Some(a) = y else { continue };`) + postdates the specification. It is scored as the `if let` it + replaces: `+1` plus the current nesting, with everything under it, + the `else` block included, one level deeper. Its `else` token does + not pay again. A `for` holding it scores 3, the same as one holding + `if let Some(a) = y { … }`. Before #1548 the `let`-`else` paid a flat + `+1` through its `else` token whatever its depth, so that `for` + scored 2. - **Python** charges a boolean operator an extra `+1` for each enclosing `lambda`, on top of the `+1` the boolean sequence itself earns. No other language does this. Only the outermost operator diff --git a/src/metrics/cognitive.rs b/src/metrics/cognitive.rs index 762c625f..6220cab7 100644 --- a/src/metrics/cognitive.rs +++ b/src/metrics/cognitive.rs @@ -3992,6 +3992,153 @@ mod tests { ); } + // A `let`-`else` is an `if let` that diverges instead of nesting the + // happy path, so it scores what its `if let` twin scores: +1 plus the + // current nesting, with a level for everything under it (#1548). + // + // Each `le_*` fn is paired with an `il_*` twin that spells the same + // control flow with `if let`, and the expected values were measured + // on the twins *before* the fix (the twins do not move). Before it, + // the let-else paid a flat +1 through the shared `else` token arm: + // `le_lp` 2, `le_d2` 4, `le_d3` 7, `le_two` 3, the closure 1 and + // `le_in` 2. `le_in` pins that the `else` block is nested like the + // twin's body (its inner `if` pays 2), `ie` that an `if … else` still + // pays its `else`, and the closure rows that the lambda level reaches + // the let-else. `value` is the fn's own score and `sum` folds in the + // closure space it encloses. + #[cfg(feature = "rust")] + #[test] + fn rust_let_else_scores_like_its_if_let_twin_1548() { + let src = "fn le_lp(xs: &[Option]) { + for y in xs { + let Some(a) = y else { continue }; + k(a); + } +} +fn il_lp(xs: &[Option]) { + for y in xs { + if let Some(a) = y { k(a); } + } +} +fn le_top(x: i32) -> i32 { + let Ok(a) = g(x) else { return 0 }; + a +} +fn il_top(x: i32) -> i32 { + if let Ok(a) = g(x) { return a; } + 0 +} +fn le_d2(xs: &[Vec>]) { + for y in xs { + for z in y { + let Some(a) = z else { continue }; + k(a); + } + } +} +fn il_d2(xs: &[Vec>]) { + for y in xs { + for z in y { + if let Some(a) = z { k(a); } + } + } +} +fn le_d3(xs: &[Vec>], c: bool) { + for y in xs { + while c { + for z in y { + let Some(a) = z else { continue }; + k(a); + } + } + } +} +fn il_d3(xs: &[Vec>], c: bool) { + for y in xs { + while c { + for z in y { + if let Some(a) = z { k(a); } + } + } + } +} +fn le_two(xs: &[Option]) { + for y in xs { + let Some(a) = y else { continue }; + let Ok(b) = g(a) else { continue }; + k(b); + } +} +fn il_two(xs: &[Option]) { + for y in xs { + if let Some(a) = y { k(a); } + if let Ok(b) = g(*y) { k(b); } + } +} +fn le_cl() { + let c = |y: Option| { + let Some(a) = y else { return 0 }; + a + }; + k(c); +} +fn il_cl() { + let c = |y: Option| { + if let Some(a) = y { return a; } + 0 + }; + k(c); +} +fn le_in(x: Option, c: bool) -> i32 { + let Some(a) = x else { + if c { return 1; } + return 0; + }; + a +} +fn il_in(x: Option, c: bool) -> i32 { + if let Some(a) = x { + if c { return a; } + } + 0 +} +fn ie(x: bool) -> i32 { + if x { 1 } else { 0 } +} +"; + // (let-else fn, if-let twin, value, sum) + let expected: &[(&str, &str, u64, u64)] = &[ + ("le_lp", "il_lp", 3, 3), + ("le_top", "il_top", 1, 1), + ("le_d2", "il_d2", 6, 6), + ("le_d3", "il_d3", 10, 10), + ("le_two", "il_two", 5, 5), + ("le_cl", "il_cl", 0, 2), + ("le_in", "il_in", 3, 3), + ("ie", "ie", 2, 2), + ]; + check_func_space::(src, "foo.rs", |space| { + assert_eq!(space.spaces.len(), 15, "fixture members"); + let measured = |n: &str| { + let c = &child_space(&space, n).metrics.cognitive; + (c.cognitive(), c.cognitive_sum()) + }; + for &(name, twin, value, sum) in expected { + assert_eq!(measured(twin), (value, sum), "{twin}"); + assert_eq!(measured(name), (value, sum), "{name}"); + } + for name in ["le_cl", "il_cl"] { + let closure = &child_space(&space, name).spaces; + assert_eq!(closure.len(), 1, "{name}'s closure space"); + assert_eq!( + closure[0].metrics.cognitive.cognitive(), + 2, + "{name} closure" + ); + } + }); + } + #[cfg(feature = "typescript")] #[test] fn typescript_if_else_if_else() { diff --git a/src/metrics/cognitive/rust.rs b/src/metrics/cognitive/rust.rs index 1f97cac7..2dc0b2f6 100644 --- a/src/metrics/cognitive/rust.rs +++ b/src/metrics/cognitive/rust.rs @@ -32,12 +32,20 @@ impl Cognitive for RustCode { ForExpression | WhileExpression | LoopExpression | MatchExpression => { increase_nesting(stats, &mut nesting); } + // `let PAT = e else { … };` is an `if let` that diverges + // instead of nesting the happy path, so it pays what its + // `if let` twin pays: a structural increment plus the current + // nesting, and a level for everything under it — the `else` + // block included, as the twin's body is (#1548). Keyed on the + // `alternative` field, as cyclomatic does (#1542). + LetDeclaration if node.child_by_field_name("alternative").is_some() => { + increase_nesting(stats, &mut nesting); + } // `Else` here is the `else` keyword token, which the grammar // also emits for the `else` of an `else if` — so this arm - // covers both. A `let`-`else` reaches it too, as a flat +1 - // with no nesting; that is a side effect of the shared token, - // not a let-else rule (#1548). - Else => { + // covers both. A `let`-`else`'s token is a direct child of the + // `let_declaration`, which the arm above already charged. + Else if !ancestors.parent_has_kind(node, LetDeclaration as u16) => { increment_by_one(stats); } BreakExpression | ContinueExpression => { From afd1475ffabcb1768e2793562ce2aec3941e8d12 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:39:09 -0700 Subject: [PATCH 23/48] fix(getter/perl): stop billing comment and regex text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tree-sitter-perl spells a comment's leading `#` as a `#` token under `comments`, and the getter listed `HASH` as an operator, so every comment billed one `#`. The grammar has one other position for that token: the pattern text of a bare `/…/`, whose `regex_pattern_content` also spells `[`, `]` and `\` as anonymous tokens. Three of the four were operators, so `/a#b[c]\#/` billed `#`, `[]` and `\` where its `m//` and `qr//` synonyms bill nothing. Drop `HASH` from the operator arm, since neither position is code, and classify every child of `regex_pattern_content` as Unknown, the compound-leaf guard the bare form's `/` delimiters already use. `$#a` and the `#` delimiters of `qw#…#`, `s#a#b#` and `q#…#` are distinct token kinds and keep their reading. A sweep of all 21 languages' commented and comment-free twins found no other comment-token leak. No corpus snapshot moves: the two Perl corpus files are not snapshotted. The getter's halstead.effort baseline entry is refreshed for the new arm. Fixes #1549 --- .bca-baseline.toml | 2 +- .rustfmt-bail-baseline.txt | 2 +- big-code-analysis-ast/src/getter/perl.rs | 15 ++++++- big-code-analysis-book/src/metrics.md | 5 ++- src/metrics/halstead.rs | 57 ++++++++++++++++++++++++ src/metrics/perl_keyword_key_tests.rs | 6 +-- 6 files changed, 80 insertions(+), 7 deletions(-) diff --git a/.bca-baseline.toml b/.bca-baseline.toml index a3d8c0da..763a31f4 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -243,7 +243,7 @@ value = 51602.8394982239 path = "big-code-analysis-ast/src/getter/perl.rs" qualified = "PerlCode::get_op_type" metric = "halstead.effort" -value = 126401.3661092431 +value = 131183.3292815811 [[entry]] path = "big-code-analysis-ast/src/getter/php.rs" diff --git a/.rustfmt-bail-baseline.txt b/.rustfmt-bail-baseline.txt index 3656a936..c9d031d0 100644 --- a/.rustfmt-bail-baseline.txt +++ b/.rustfmt-bail-baseline.txt @@ -135,7 +135,7 @@ big-code-analysis-ast/src/getter/java.rs 5 big-code-analysis-ast/src/getter/kotlin.rs 4 big-code-analysis-ast/src/getter/lua.rs 3 big-code-analysis-ast/src/getter/mozcpp.rs 4 -big-code-analysis-ast/src/getter/perl.rs 3 +big-code-analysis-ast/src/getter/perl.rs 4 big-code-analysis-ast/src/getter/php.rs 11 big-code-analysis-ast/src/getter/python.rs 7 big-code-analysis-ast/src/getter/ruby.rs 4 diff --git a/big-code-analysis-ast/src/getter/perl.rs b/big-code-analysis-ast/src/getter/perl.rs index acbc949c..c3a99095 100644 --- a/big-code-analysis-ast/src/getter/perl.rs +++ b/big-code-analysis-ast/src/getter/perl.rs @@ -44,6 +44,19 @@ impl Getter for PerlCode { { TokenRole::Unknown } + // The bare form's pattern text is one operand, billed on the + // `PatternMatcher` wrapper, but the grammar spells four of its + // characters as anonymous tokens under `regex_pattern_content`: + // `[`, `]`, `\` and `#`. They are literal regex text, which + // `m//` and `qr//` emit no node for, so `/[a]/` billed a `[]` + // operator that `m/[a]/` does not (#1549). `#` is also in no + // operator arm, because its only other position is a comment's + // leading `#` under `comments`, and a comment is not code. + // `$#a` and the `#` delimiters of `qw#…#` / `s#a#b#` are + // distinct token kinds. + _ if ancestors.parent_has_kind(node, P::RegexPatternContent as u16) => { + TokenRole::Unknown + } // Control-flow and declaration keywords. `Perl::Sub` is the // `sub` keyword (token id 16); `Perl::SUB` is the `__SUB__` // literal (token id 7) — that one is an operand, not an @@ -57,7 +70,7 @@ impl Getter for PerlCode { // Punctuation acting as operators | P::SEMI | P::COMMA | P::COLON | P::COLONCOLON | P::LBRACE | P::LBRACK | P::LPAREN | P::DOT | P::DOTDOT | P::DOTDOTDOT | P::FatComma | P::DASHGT - | P::QMARK | P::BSLASH | P::DOLLAR | P::DOLLARHASH | P::AT | P::PERCENT | P::HASH + | P::QMARK | P::BSLASH | P::DOLLAR | P::DOLLARHASH | P::AT | P::PERCENT // Arithmetic / comparison / logical / bitwise / assignment operators | P::EQ | P::PLUS | P::DASH | P::STAR | P::SLASH | P::STARSTAR | P::BANG | P::TILDE | P::EQTILDE | P::BANGTILDE | P::EQEQ | P::BANGEQ | P::LT | P::GT diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 27db2f47..665b7b7c 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -622,7 +622,10 @@ tokens involved the same way they spell real operators: and a Perl or Ruby pattern each contribute *one operand* — the literal — and no operator for the punctuation around it. Otherwise the score would move with the author's choice of delimiter, which - says nothing about the code. Tcl and iRules need one extra step to + says nothing about the code. The same holds for punctuation the + grammar spells *inside* a literal: Perl's bare `/[a]#/` bills no `[]` + or `#`, matching `m/[a]#/`, whose text the grammar does not split + into tokens (#1549). Tcl and iRules need one extra step to honour this, because their grammars spell a braced literal and a braced script body the same way: whether `{a b}` is a block or a quoted value depends on the command it is passed to, so the diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index d5e666e5..58267b7a 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -4397,6 +4397,63 @@ mod tests { } } + #[cfg(feature = "perl")] + #[test] + fn perl_comment_leader_is_not_an_operator_1549() { + // tree-sitter-perl emits a comment's leading `#` as a `#` token + // under `comments`, and the getter listed it as an operator, so + // every comment billed one `#` (#1549). The oracle is the same + // source with its comments deleted, which must score identically. + // `$#a` and `qw#…#` are kept to pin that dropping `#` leaves the + // distinct `$#` operator and the `qw` delimiters alone. + // + // expected: operators `sub`, `{}`, `my`, `=`, `;` × 2, `return`, + // `$#` × 2, `+` → n1 = 8, N1 = 10. Operands `f`, `@a`, `x`, `y`, + // `a` × 2 → n2 = 5, N2 = 6. Before the fix the commented row + // added `#` → n1 = 9, N1 = 13. Once excluded, a comment feeds no + // axis to anchor on, so only the revert run guards the fixture: + // keep its comments. + for (source, label) in [ + ( + "sub f {\n # leading\n my @a = qw#x y#; # tail\n return $#a + $#a; # tail\n}\n", + "commented", + ), + ( + "sub f {\n my @a = qw#x y#;\n return $#a + $#a;\n}\n", + "comment-free twin", + ), + ] { + assert_halstead_counts::(source, "foo.pl", [8, 10, 5, 6], label); + } + } + + #[cfg(feature = "perl")] + #[test] + fn perl_bare_pattern_text_bills_no_operator_1549() { + // The bare `/…/` form keeps its pattern text in a + // `regex_pattern_content` that spells `#`, `[`, `]` and `\` as + // anonymous tokens, three of which an operator arm listed, so + // `/a#b[c]\#/` billed `#`, `[]` and `\` operators. `m//` and + // `qr//` emit no node for the same text, so they are the oracle: + // all three spellings must score as `/abc/` does in + // `perl_every_pattern_value_spelling_scores_alike`. + // + // expected per variant: operators `$` × 2, `=~` × 2, `and`, `;` + // → n1 = 4, N1 = 6; operands `$s` × 2 and the pattern × 2 → + // n2 = 2, N2 = 4. Before the fix the bare row scored n1 = 7, + // N1 = 14. The `#`, `[c]` and `\#` feed no axis once excluded, + // so nothing anchors them and only the revert run guards the + // fixture: keep them in the bare row. + for pattern in [r"/a#b[c]\#/", r"m/a#b[c]\#/", r"qr/a#b[c]\#/"] { + assert_halstead_counts::( + &format!("$s =~ {pattern} and $s =~ {pattern};\n"), + "foo.pl", + [4, 6, 2, 4], + &format!("pattern {pattern}"), + ); + } + } + #[cfg(feature = "perl")] #[test] fn perl_every_pattern_operation_spelling_scores_alike() { diff --git a/src/metrics/perl_keyword_key_tests.rs b/src/metrics/perl_keyword_key_tests.rs index 4f1f9f78..0ec5d421 100644 --- a/src/metrics/perl_keyword_key_tests.rs +++ b/src/metrics/perl_keyword_key_tests.rs @@ -323,13 +323,13 @@ const DASH_KEY_ROWS: [(&str, &str, [u64; 4], Halstead); 18] = [ [2, 0, 0, 0], [6, 8, 4, 4], ), - // A comment in the subscript. Its `#` bills as an operator in both - // spellings (#1549). + // A comment in the subscript bills nothing, so this scores as the + // row above. Its `#` billed an operator until #1549 (7, 9). ( "my %h; return $h{'-foo' # why\n};", "my %h; return $h{-foo # why\n};", [2, 0, 0, 0], - [7, 9, 4, 4], + [6, 8, 4, 4], ), ( "my $r; return $r->{'-foo'};", From c2505349c2a482da6744d7ba4a983e879938cfc2 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:54:25 -0700 Subject: [PATCH 24/48] test(concurrent): rendezvous before counting consumers `num_jobs_is_the_consumer_count_not_a_budget_shared_with_a_producer` counted the distinct threads that had run the callback, a read taken at one point in time. Under scheduler pressure the second consumer can start only after the first has drained all 200 files, so it existed but was never seen, and the test failed with "got 1". Production is not under-provisioned: `run` spawns exactly `num_jobs` consumers on one shared receiver. Each consumer now waits on its first file until both have arrived. With the first parked, the second is guaranteed a file whenever it starts. The wait is bounded at 10 s, so the pre-#1114 single consumer still fails the "more than one" assertion rather than hanging. The 50 us sleep that only made the race less likely is gone. The flake did not reproduce naturally: 600 isolated runs under CPU load and 55 full-suite runs all passed. A 50 ms start delay injected into consumer 1 makes it deterministic: the old test failed 184/200 under load (200/200 pinned to one CPU), the new one passed 200/200 in both, and it passes 200/200 under load without the delay. Fixes #1519 --- src/concurrent_files_tests.rs | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/concurrent_files_tests.rs b/src/concurrent_files_tests.rs index 79ec4fd0..62d9cd8e 100644 --- a/src/concurrent_files_tests.rs +++ b/src/concurrent_files_tests.rs @@ -470,13 +470,20 @@ fn without_path_verification_dispatches_every_path() { /// only way to observe the difference from outside: dispatch itself now /// happens on this thread. /// -/// Asserted as an upper bound plus "more than one", because the pool is -/// work-stealing: with few files a fast consumer can drain the channel -/// before its peers wake, so the exact count is not deterministic. The -/// old `num_jobs - 1` behaviour is still excluded — at `num_jobs = 2` it -/// permitted exactly one thread, and this fixture reaches two. +/// The count is taken behind a rendezvous: each consumer, on its first +/// file, waits for the other before returning. Counting the threads +/// that happened to run the callback is a point-in-time read: under +/// scheduler pressure the second consumer can start after the first has +/// drained all 200 files, so it existed but was never seen (#1519). With +/// the first consumer parked, the second is guaranteed a file whenever +/// it starts. The wait is bounded so that the pre-#1114 single consumer +/// fails the "more than one" assertion below instead of hanging. #[test] fn num_jobs_is_the_consumer_count_not_a_budget_shared_with_a_producer() { + // Generous enough for any scheduler delay a loaded CI host imposes; + // it is only ever waited out in full when a consumer is missing. + const RENDEZVOUS_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + let tmp = Builder::new().prefix("jobs").tempdir().expect("tempdir"); let root = tmp.path(); let mut paths = Vec::new(); @@ -488,14 +495,24 @@ fn num_jobs_is_the_consumer_count_not_a_budget_shared_with_a_producer() { let threads = Arc::new(Mutex::new(std::collections::HashSet::new())); let sink = Arc::clone(&threads); + let gate = (Mutex::new(0_usize), std::sync::Condvar::new()); let caller = thread::current().id(); ConcurrentRunner::new(2, move |_path: PathBuf, _cfg: &()| { - sink.lock() + let first_file = sink + .lock() .expect("uncontended in test") .insert(thread::current().id()); - // Hold the worker briefly so the second consumer is given a - // chance to pick up work rather than losing every race. - thread::sleep(std::time::Duration::from_micros(50)); + if first_file { + let (count, ready) = &gate; + let mut count = count.lock().expect("uncontended in test"); + *count += 1; + ready.notify_all(); + drop( + ready + .wait_timeout_while(count, RENDEZVOUS_TIMEOUT, |n| *n < 2) + .expect("uncontended in test"), + ); + } Ok(()) }) .run((), FilesData { paths }) From bd945c3432c4d5d656a3856ed36515b8b4541a8a Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:05:29 -0700 Subject: [PATCH 25/48] fix(halstead): bill Python and Rust punctuation #1395 made punctuation and delimiters Halstead vocabulary, but two getters still left glyphs out: Python billed only `,` and `.`, and Rust billed `::` but not `:`. Python now bills `(`, `[`, `{` (once per pair, rendered `()` / `[]` / `{}` like its siblings), `;` and every syntactic `:`. An f-string's replacement-field `{` and format-spec `:` stay unbilled, matching the #1314 interpolation-opener rule. Rust now bills its single `:`, a separate entry from `::`. Every Python and Rust Halstead score, and the MI derived from it, rises. 116 of 172 serde corpus snapshots move (Halstead and MI only); the self-scan baseline is refreshed for the repo's own Rust code. Fixes #1486 --- .bca-baseline.toml | 146 +++++--- big-code-analysis-ast/src/getter.rs | 8 +- big-code-analysis-ast/src/getter/python.rs | 40 ++- big-code-analysis-ast/src/getter/rust.rs | 6 +- big-code-analysis-book/src/metrics.md | 21 +- .../tests/check/check_explain_threshold.rs | 12 +- big-code-analysis-web/src/web/server_tests.rs | 150 ++++---- src/metrics/halstead.rs | 330 +++++++++++++----- src/metrics/mi.rs | 6 +- src/ops.rs | 22 +- ...formats__csv_test__csv_python_greeter.snap | 8 +- ...t_formats__csv_test__csv_rust_counter.snap | 2 +- tests/repositories/big-code-analysis-output | 2 +- 13 files changed, 517 insertions(+), 236 deletions(-) diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 763a31f4..8b89e54a 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -189,19 +189,19 @@ value = 6.0 path = "big-code-analysis-ast/src/ast.rs" qualified = "build" metric = "halstead.effort" -value = 87299.1671815905 +value = 95280.22256472526 [[entry]] path = "big-code-analysis-ast/src/c_macro.rs" qualified = "replace" metric = "halstead.effort" -value = 82074.58162485641 +value = 87076.53005268889 [[entry]] path = "big-code-analysis-ast/src/c_macro.rs" qualified = "step_normal" metric = "halstead.effort" -value = 66515.67234293568 +value = 70691.13308026636 [[entry]] path = "big-code-analysis-ast/src/c_macro.rs" @@ -221,47 +221,53 @@ qualified = "step_raw_string" metric = "nargs" value = 5.0 +[[entry]] +path = "big-code-analysis-ast/src/getter.rs" +qualified = "Getter::is_braced_literal_slot" +metric = "halstead.effort" +value = 50566.900032069454 + [[entry]] path = "big-code-analysis-ast/src/getter/bash.rs" qualified = "BashCode::get_op_type" metric = "halstead.effort" -value = 57421.05047463073 +value = 61150.53156222458 [[entry]] path = "big-code-analysis-ast/src/getter/elixir.rs" qualified = "ElixirCode::get_op_type" metric = "halstead.effort" -value = 76992.47971865896 +value = 81284.21083173114 [[entry]] path = "big-code-analysis-ast/src/getter/irules.rs" qualified = "IrulesCode::get_op_type" metric = "halstead.effort" -value = 51602.8394982239 +value = 54835.65543871385 [[entry]] path = "big-code-analysis-ast/src/getter/perl.rs" qualified = "PerlCode::get_op_type" metric = "halstead.effort" -value = 131183.3292815811 +value = 137407.31980126494 [[entry]] path = "big-code-analysis-ast/src/getter/php.rs" qualified = "PhpCode::get_op_type" metric = "halstead.effort" -value = 92711.94569253008 +value = 96592.02057608281 [[entry]] path = "big-code-analysis-ast/src/getter/python.rs" qualified = "PythonCode::get_op_type" metric = "halstead.effort" -value = 48346.66496041094 +value = 64442.20035097394 [[entry]] path = "big-code-analysis-ast/src/getter/ruby.rs" qualified = "RubyCode::get_op_type" metric = "halstead.effort" -value = 120805.48245794396 +value = 127317.70075099175 [[entry]] path = "big-code-analysis-ast/src/node.rs" @@ -273,7 +279,7 @@ value = 36.0 path = "big-code-analysis-ast/src/parser.rs" qualified = "Parser::filters" metric = "halstead.effort" -value = 85508.52631578948 +value = 92491.6833558545 [[entry]] path = "big-code-analysis-ast/src/preproc.rs" @@ -303,19 +309,19 @@ value = 7.0 path = "big-code-analysis-cli/src/baseline.rs" qualified = "Baseline::from_str" metric = "halstead.effort" -value = 58410.89968953587 +value = 62587.95570171439 [[entry]] path = "big-code-analysis-cli/src/baseline_diff.rs" qualified = "BaselineDiff::compute" metric = "halstead.effort" -value = 118801.67275912332 +value = 124425.17843774539 [[entry]] path = "big-code-analysis-cli/src/check_format.rs" qualified = "write_github_annotations" metric = "halstead.effort" -value = 47834.17129195417 +value = 51030.97459989012 [[entry]] path = "big-code-analysis-cli/src/commands.rs" @@ -327,13 +333,19 @@ value = 136001.1105471366 path = "big-code-analysis-cli/src/commands/analyze.rs" qualified = "run_command_metrics" metric = "halstead.effort" -value = 69608.24451083591 +value = 74146.94504520179 + +[[entry]] +path = "big-code-analysis-cli/src/commands/analyze.rs" +qualified = "run_command_ops" +metric = "halstead.effort" +value = 49966.059463762176 [[entry]] path = "big-code-analysis-cli/src/commands/analyze.rs" qualified = "write_aggregate" metric = "halstead.effort" -value = 54210.895759662584 +value = 57901.393538811906 [[entry]] path = "big-code-analysis-cli/src/commands/check.rs" @@ -357,7 +369,7 @@ value = 6.0 path = "big-code-analysis-cli/src/commands/check.rs" qualified = "run_check" metric = "halstead.effort" -value = 63636.29428662941 +value = 67343.38425539367 [[entry]] path = "big-code-analysis-cli/src/commands/check/effective_config.rs" @@ -405,13 +417,13 @@ value = 39.06404996924922 path = "big-code-analysis-cli/src/commands/report.rs" qualified = "run_command_report" metric = "halstead.effort" -value = 63112.954051595756 +value = 67793.18512453695 [[entry]] path = "big-code-analysis-cli/src/diff.rs" qualified = "materialize_tree" metric = "halstead.effort" -value = 60390.01093335322 +value = 63004.79514502588 [[entry]] path = "big-code-analysis-cli/src/diff.rs" @@ -503,6 +515,12 @@ qualified = "dispatch_strip_comments" metric = "nargs" value = 6.0 +[[entry]] +path = "big-code-analysis-cli/src/dispatch.rs" +qualified = "validate_and_resolve_file" +metric = "halstead.effort" +value = 48161.176757774374 + [[entry]] path = "big-code-analysis-cli/src/html_report.rs" qualified = "generate_html_report_with_vcs" @@ -555,7 +573,13 @@ value = 7.0 path = "big-code-analysis-cli/src/lib.rs" qualified = "legacy_hint" metric = "halstead.effort" -value = 59351.805921378844 +value = 62321.99298455441 + +[[entry]] +path = "big-code-analysis-cli/src/manifest.rs" +qualified = "Manifest::merge_check" +metric = "halstead.effort" +value = 49443.19551903411 [[entry]] path = "big-code-analysis-cli/src/markdown_report.rs" @@ -615,13 +639,13 @@ value = 7.0 path = "big-code-analysis-cli/src/threshold_suggestion.rs" qualified = "closest_names" metric = "halstead.effort" -value = 55785.628204736706 +value = 58505.309528447164 [[entry]] path = "big-code-analysis-cli/src/threshold_suggestion.rs" qualified = "edit_distance_with_cutoff" metric = "halstead.effort" -value = 66737.04848699087 +value = 70743.85391258422 [[entry]] path = "big-code-analysis-cli/src/thresholds.rs" @@ -825,7 +849,7 @@ value = 747.0 path = "big-code-analysis-py/src/vcs.rs" qualified = "options_from" metric = "halstead.effort" -value = 47661.70050580648 +value = 50629.47325240143 [[entry]] path = "big-code-analysis-py/src/vcs.rs" @@ -845,11 +869,17 @@ qualified = "walk_seed" metric = "nargs" value = 5.0 +[[entry]] +path = "big-code-analysis-web/src/web/negotiate.rs" +qualified = "negotiate" +metric = "halstead.effort" +value = 49354.63340093155 + [[entry]] path = "big-code-analysis-web/src/web/server.rs" qualified = "run_parse" metric = "halstead.effort" -value = 82478.1543040951 +value = 87922.2135069964 [[entry]] path = "big-code-analysis-web/src/web/server.rs" @@ -873,7 +903,7 @@ value = 5.0 path = "big-code-analysis-web/src/web/vcs.rs" qualified = "options_from" metric = "halstead.effort" -value = 51846.28098938873 +value = 55077.016098147826 [[entry]] path = "big-code-analysis-web/src/web/vcs.rs" @@ -887,6 +917,12 @@ qualified = "KotlinCode::compute" metric = "cyclomatic" value = 20.0 +[[entry]] +path = "src/metrics/abc/ruby.rs" +qualified = "RubyCode::compute" +metric = "halstead.effort" +value = 48099.42954412001 + [[entry]] path = "src/metrics/abc/rust.rs" qualified = "RustCode::compute" @@ -1029,7 +1065,13 @@ value = 5.0 path = "src/metrics/loc/perl.rs" qualified = "PerlCode::compute" metric = "halstead.effort" -value = 51486.205050501565 +value = 54607.47985931332 + +[[entry]] +path = "src/metrics/npa/php.rs" +qualified = "PhpCode::compute" +metric = "halstead.effort" +value = 50326.64993163575 [[entry]] path = "src/metrics/npa/python.rs" @@ -1041,7 +1083,7 @@ value = 6.0 path = "src/ops.rs" qualified = "ops_inner" metric = "halstead.effort" -value = 67661.84803230887 +value = 73118.98435042621 [[entry]] path = "src/output/checkstyle.rs" @@ -1053,7 +1095,7 @@ value = 7.0 path = "src/output/code_climate.rs" qualified = "write_code_climate" metric = "halstead.effort" -value = 60596.13916454601 +value = 65397.75866742601 [[entry]] path = "src/output/dump.rs" @@ -1125,7 +1167,7 @@ value = 111.87939935484101 path = "src/output/funcspace_row.rs" qualified = "metric_values" metric = "halstead.effort" -value = 80351.11907066956 +value = 86304.11475310498 [[entry]] path = "src/spaces.rs" @@ -1155,7 +1197,7 @@ value = 5.0 path = "src/spaces/compute.rs" qualified = "compute_per_node" metric = "halstead.effort" -value = 63007.70094399036 +value = 68533.65287917512 [[entry]] path = "src/spaces/compute.rs" @@ -1167,7 +1209,7 @@ value = 7.0 path = "src/spaces/compute.rs" qualified = "metrics_inner" metric = "halstead.effort" -value = 116715.6053686016 +value = 123785.03296627598 [[entry]] path = "src/spaces/compute.rs" @@ -1185,7 +1227,7 @@ value = 6.0 path = "src/vcs/bus_factor.rs" qualified = "compute" metric = "halstead.effort" -value = 47694.803479864124 +value = 51947.0591918485 [[entry]] path = "src/vcs/bus_factor.rs" @@ -1193,6 +1235,12 @@ qualified = "greedy_truck_factor" metric = "cognitive" value = 15.0 +[[entry]] +path = "src/vcs/bus_factor.rs" +qualified = "greedy_truck_factor" +metric = "halstead.effort" +value = 47960.96527182218 + [[entry]] path = "src/vcs/bus_factor.rs" qualified = "greedy_truck_factor" @@ -1227,7 +1275,7 @@ value = 6.0 path = "src/vcs/git/cached.rs" qualified = "build_cached" metric = "halstead.effort" -value = 74812.5424082547 +value = 78921.96719178827 [[entry]] path = "src/vcs/git/cached.rs" @@ -1257,13 +1305,13 @@ value = 6.0 path = "src/vcs/git/diff_parse.rs" qualified = "unquote_git_path" metric = "halstead.effort" -value = 71494.28221151498 +value = 74373.33475054632 [[entry]] path = "src/vcs/git/history.rs" qualified = "collect_events" metric = "halstead.effort" -value = 48928.29416311421 +value = 52128.25957334215 [[entry]] path = "src/vcs/git/history.rs" @@ -1301,6 +1349,12 @@ qualified = "process_commit" metric = "nexits" value = 9.0 +[[entry]] +path = "src/vcs/git/jit.rs" +qualified = "collect_touched" +metric = "halstead.effort" +value = 49957.782203054114 + [[entry]] path = "src/vcs/git/jit.rs" qualified = "collect_touched" @@ -1341,7 +1395,7 @@ value = 6.0 path = "src/vcs/git/jit.rs" qualified = "history_features" metric = "halstead.effort" -value = 52560.02538757721 +value = 56372.217728048636 [[entry]] path = "src/vcs/git/jit.rs" @@ -1379,6 +1433,12 @@ qualified = "build_trend" metric = "nexits" value = 5.0 +[[entry]] +path = "src/vcs/jit.rs" +qualified = "score" +metric = "halstead.effort" +value = 48585.950863463186 + [[entry]] path = "src/vcs/options.rs" qualified = "parse_iso8601" @@ -1397,23 +1457,29 @@ qualified = "fold_commit" metric = "nargs" value = 5.0 +[[entry]] +path = "src/vcs/replay.rs" +qualified = "replay" +metric = "halstead.effort" +value = 49393.80551407597 + [[entry]] path = "src/vcs/score.rs" qualified = "apply_percentile" metric = "halstead.effort" -value = 68450.87280037583 +value = 71674.67451821786 [[entry]] path = "src/vcs/stats.rs" qualified = "Accumulator::finalize" metric = "halstead.effort" -value = 79569.58843331039 +value = 89416.15847479667 [[entry]] path = "src/vcs/stats.rs" qualified = "Accumulator::record" metric = "halstead.effort" -value = 56240.27018612609 +value = 59083.201813811575 [[entry]] path = "src/vcs/trend.rs" @@ -1425,4 +1491,4 @@ value = 5.0 path = "src/wire.rs" qualified = "CodeMetrics::from" metric = "halstead.effort" -value = 49790.15156224738 +value = 55821.36704417831 diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index 69b462b7..b9f01a26 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -502,11 +502,9 @@ pub trait Getter { /// opens a JSX tag, brackets a Lua `` attribute, names a C# /// `operator <` overload, disambiguates a Kotlin `super`, or /// delimits a Perl `` readline. Every impl that classifies - /// punctuation at all already bills `(`, `{`, `[`, `,`, `;`, `.` - /// and `:` on exactly that reading, for each glyph its language - /// spells — bar two, which #1395 did not revisit: Python - /// classifies only `,` and `.` of that set, and Rust bills `::` - /// but not the type-annotation `:`. + /// punctuation at all bills `(`, `{`, `[`, `,`, `;`, `.` and `:` on + /// exactly that reading, for each glyph its language spells. Python + /// and Rust were the last two exceptions, closed by #1486. /// /// A *decision* metric gates the same token by role, and is /// supposed to disagree: ABC's `conditions` counts a `<` only under diff --git a/big-code-analysis-ast/src/getter/python.rs b/big-code-analysis-ast/src/getter/python.rs index f038eaad..7179e7ac 100644 --- a/big-code-analysis-ast/src/getter/python.rs +++ b/big-code-analysis-ast/src/getter/python.rs @@ -3,6 +3,27 @@ use super::*; +/// The role of a `{` or `:`, which Python spells both as syntax and +/// inside an f-string's replacement field. +/// +/// As syntax each is an operator (#1486). The interpolation opener +/// (`f"{x}"`, and a nested `{w}` inside a format spec) is spelling, not +/// an operation — the six interpolating languages settled that in #1314 +/// and Python follows them. The `:` that opens a format spec +/// (`f"{x:>10}"`) introduces the format mini-language, whose text is +/// string content and counts as nothing, so its delimiter does not +/// count either. +fn brace_or_colon_role(kind: Python, parent: Option) -> TokenRole { + use Python::*; + + match (kind, parent) { + (LBRACE, Some(Interpolation | FormatExpression)) | (COLON, Some(FormatSpecifier)) => { + TokenRole::Unknown + } + _ => TokenRole::Operator, + } +} + impl Getter for PythonCode { fn get_space_kind(node: &Node) -> SpaceKind { match node.kind_id().into() { @@ -16,7 +37,17 @@ impl Getter for PythonCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Python::*; - match node.kind_id().into() { + let kind: Python = node.kind_id().into(); + // Bracket openers, `;` and `:` are vocabulary like every other + // getter's (#1395, #1486); only the f-string replacement-field + // spellings are withdrawn. An early return rather than an arm, + // because the match below is one rustfmt declines to format. + if matches!(kind, LBRACE | COLON) { + let parent = ancestors.parent(node).map(|p| p.kind_id().into()); + return brace_or_colon_role(kind, parent); + } + + match kind { // The `not` / `in` / `is` leaf tokens are operators on their own // (`not x`, `a in b`, `a is b`, `for x in y`), but the grammar // also nests them inside the compound `not in` (Notin) and @@ -34,7 +65,8 @@ impl Getter for PythonCode { | AT | And | Or | PLUS | DASH | SLASH | PERCENT | SLASHSLASH | STARSTAR | PIPE | AMP | CARET | LTLT | TILDE | LT | LTEQ | EQEQ | BANGEQ | GTEQ | GT | LTGT | PLUSEQ | DASHEQ | STAREQ | SLASHEQ | ATEQ | SLASHSLASHEQ | PERCENTEQ | STARSTAREQ - | GTGTEQ | LTLTEQ | AMPEQ | CARETEQ | PIPEEQ | Yield | Print + | GTGTEQ | LTLTEQ | AMPEQ | CARETEQ | PIPEEQ | Yield | Print | LPAREN | LBRACK + | SEMI // `not in` / `is not` compounds count as one operator each; the // inner Not/In/Is leaves are suppressed by the parent-guard arm // above (#413). @@ -92,7 +124,5 @@ impl Getter for PythonCode { } } - fn get_operator_id_as_str(id: u16) -> &'static str { - Into::::into(id).into() - } + get_operator!(Python); } diff --git a/big-code-analysis-ast/src/getter/rust.rs b/big-code-analysis-ast/src/getter/rust.rs index bce73cd0..7eea8c29 100644 --- a/big-code-analysis-ast/src/getter/rust.rs +++ b/big-code-analysis-ast/src/getter/rust.rs @@ -76,9 +76,9 @@ impl Getter for RustCode { | PrimitiveType3 | PrimitiveType4 | PrimitiveType5 | PrimitiveType6 | PrimitiveType7 | PrimitiveType8 | PrimitiveType9 | PrimitiveType10 | PrimitiveType11 | PrimitiveType12 | PrimitiveType13 | PrimitiveType14 - | PrimitiveType15 | PrimitiveType16 | PrimitiveType17 | Fn | SEMI | COLONCOLON - | Const | Static | Enum | Struct | Trait | Impl | Use | Mod | Pub | Type | Union - | Where | Extern | Dyn => TokenRole::Operator, + | PrimitiveType15 | PrimitiveType16 | PrimitiveType17 | Fn | SEMI | COLON + | COLONCOLON | Const | Static | Enum | Struct | Trait | Impl | Use | Mod | Pub + | Type | Union | Where | Extern | Dyn => TokenRole::Operator, // FieldIdentifier (e.g. `p.x`) and TypeIdentifier (e.g. `Vec`, // `HashMap`) are operand-class names — C++ and Go classify them // the same way (see arms ~588 and ~862 below). Omitting them diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 665b7b7c..44770ed0 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -609,9 +609,12 @@ notation for the same four counts. big-code-analysis records these four numbers in `src/metrics/halstead.rs` per function and per file. The per-language trait classifies tokens as operator vs. operand on a token-by-token -basis; the rules deliberately exclude pure layout punctuation like -parentheses and statement separators, which is why the Halstead -totals are *not* the same as the Tokens count. +basis. Punctuation counts as an operator: a bracket pair is one +operator, billed once at its opener and listed by `bca ops` as `()`, +`[]` or `{}`, and `,`, `;`, `.` and `:` count once per occurrence. +Closing brackets, comments and a literal's own delimiters count as +nothing, which is why the Halstead totals are *not* the same as the +Tokens count. Seven classification rules are worth knowing because they are choices rather than consequences, and because several grammars spell the @@ -684,9 +687,11 @@ tokens involved the same way they spell real operators: belongs to it — so the list would be incomplete by construction and would make three behaviours where there are two. - **A string-interpolation opener is not an operator.** `"{$x}"` in - PHP, `"#{x}"` in Ruby and Elixir, `"${x}"` in Kotlin and Groovy and - `$"{x}"` in C# all count the interpolated expression's own operators - and nothing for the opener itself. + PHP, `"#{x}"` in Ruby and Elixir, `"${x}"` in Kotlin and Groovy, + `$"{x}"` in C# and `f"{x}"` in Python all count the interpolated + expression's own operators and nothing for the opener itself. The + `:` that opens a Python format spec (`f"{x:>10}"`) counts nothing + either, because the spec after it is string content (#1486). - **A delimiter around *syntax* is an operator, whatever it delimits.** The rule above is about a literal's own punctuation, and it stops there. Halstead counts the vocabulary a program is written in and has @@ -699,6 +704,10 @@ tokens involved the same way they spell real operators: tokens from `conditions`: ABC asks whether a branch is taken, and these are not branches; Halstead asks what alphabet the source uses, and these are part of it. Expect the two metrics to disagree here. + The same reading bills Python's `(`, `[`, `{`, `;` and block, + slice, dict and annotation `:`, and Rust's annotation, field, bound + and label `:`, which is a separate operator from `::`. Both languages + left these out until #1486, so their Halstead scores rose with it. - **An alternative spelling is an operator of its own.** C++ spells `&&` equally as `and`, `!=` as `not_eq` and so on through all eleven [alternative tokens](https://en.cppreference.com/w/cpp/language/operator_alternative). diff --git a/big-code-analysis-cli/tests/check/check_explain_threshold.rs b/big-code-analysis-cli/tests/check/check_explain_threshold.rs index 0e32cb35..8fde35c6 100644 --- a/big-code-analysis-cli/tests/check/check_explain_threshold.rs +++ b/big-code-analysis-cli/tests/check/check_explain_threshold.rs @@ -523,16 +523,16 @@ fn soft_limit_is_derived_by_metric_direction() { /// expression, `breaches_limit(v.value, ceiling, v.lower_is_worse)`. /// /// The test above cannot reach it: `mi.original=20` is a floor that every -/// one-line fixture function (all measure ≈147-149) clears, so its +/// one-line fixture function (all measure ≈145-148) clears, so its /// population is empty at both tiers and the partition never runs. /// Hard-coding `false` there — the pre-#1166 "higher is worse" assumption /// — failed none of the suite's 5053 tests. /// -/// A floor of `147.5` splits the fixture's three measured values -/// (146.9456 ×3, 147.7445 ×6, 148.655 ×20): three sit below it and -/// breach, all 29 sit below the derived `155.264` soft floor. Read with +/// A floor of `146` splits the fixture's three measured values +/// (145.7045 ×3, 146.5454 ×6, 147.5134 ×20): three sit below it and +/// breach, all 29 sit below the derived `153.684` soft floor. Read with /// the direction inverted the hard tier would instead collect the 26 -/// *above* 147.5, so the two readings share no count. +/// *above* 146, so the two readings share no count. /// /// Both metrics are explained in one run, and both carry a *non-empty* /// population — which is the second thing this pins. Every other @@ -551,7 +551,7 @@ fn a_lower_is_worse_metric_partitions_offenders_by_direction() { "--paths", "lib.rs", "--explain-threshold", - "mi.original=147.5", + "mi.original=146", "--explain-threshold", &candidate, ])); diff --git a/big-code-analysis-web/src/web/server_tests.rs b/big-code-analysis-web/src/web/server_tests.rs index 33e868e7..1bb9c1d3 100644 --- a/big-code-analysis-web/src/web/server_tests.rs +++ b/big-code-analysis-web/src/web/server_tests.rs @@ -743,26 +743,26 @@ async fn test_web_metrics_json() { "cognitive": {"sum": 0, "value": 0, "average": 0.0, "min":0, "max":0}, "nargs": {"function_args": 0, "function_args_average": 0.0, "closure_args": 0, "closure_args_average": 0.0, "total": 0, "value": 0, "average": 0.0, "closure_args_max": 0, "closure_args_min": 0, "function_args_max": 0, "function_args_min": 0}, "nexits": {"sum": 0, "average": 0.0,"min":0,"max":0}, - "halstead": {"bugs": 0.000_942_552_557_372_941_4, - "difficulty": 1.0, - "effort": 4.754_887_502_163_468, - "length": 3, - "estimated_program_length": 2.0, - "purity_ratio": 0.666_666_666_666_666_6, - "level": 1.0, + "halstead": {"bugs": 0.002_712_967_490_108_627, + "difficulty": 2.0, + "effort": 23.219_280_948_873_62, + "length": 5, + "estimated_program_length": 8.0, + "purity_ratio": 1.6, + "level": 0.5, "total_operands": 1, - "total_operators": 2, - "vocabulary": 3, - "time": 0.264_160_416_786_859_36, + "total_operators": 4, + "vocabulary": 5, + "time": 1.289_960_052_715_201, "unique_operands": 1, - "unique_operators": 2, - "volume": 4.754_887_502_163_468}, + "unique_operators": 4, + "volume": 11.609_640_474_436_81}, "loc": {"cloc": 1, "ploc": 2, "lloc": 1, "sloc": 4, "blank": 1, "cloc_average": 0.5, "ploc_average": 1.0, "lloc_average": 0.5, "sloc_average": 2.0, "blank_average": 0.5, "cloc_min": 0, "ploc_min": 2, "lloc_min": 1, "sloc_min": 2, "blank_min": 0, "cloc_max": 1, "ploc_max": 2, "lloc_max": 1, "sloc_max": 4, "blank_max": 1}, "nom": {"functions": 1, "closures": 0, "functions_average": 0.5, "closures_average": 0.0, "total": 1, "average": 0.5, "closures_min": 0, "closures_max": 0, "functions_min": 0, "functions_max": 1}, "tokens": {"tokens": 6, "average": 3.0, "min": 0, "max": 6}, - "mi": {"original": 139.974_331_558_152_1, - "sei": 176.151_664_509_301_33, - "visual_studio": 81.856_334_244_533_39}, + "mi": {"original": 135.332_485_042_474_81, + "sei": 169.454_895_560_566, + "visual_studio": 79.141_804_118_406_33}, "abc": {"assignments": 0, "branches": 0, "conditions": 0, "magnitude": 0.0, "value": 0.0, "assignments_average": 0.0, "branches_average": 0.0, "conditions_average": 0.0, "assignments_min": 0, "assignments_max": 0, "branches_min": 0, "branches_max": 0, "conditions_min": 0, "conditions_max": 0}, "wmc": {"class_wmc_sum": 0, "interface_wmc_sum": 0, "total": 0}, "npm": {"class_npm_sum": 0, "interface_npm_sum": 0, "class_methods": 0, "interface_methods": 0, "class_coa": 0.0, "interface_coa": 0.0, "total": 0, "total_methods": 0, "coa": 0.0}, @@ -775,26 +775,26 @@ async fn test_web_metrics_json() { "cognitive": {"sum": 0, "value": 0, "average": 0.0 , "min":0, "max":0}, "nargs": {"function_args": 0, "function_args_average": 0.0, "closure_args": 0, "closure_args_average": 0.0, "total": 0, "value": 0, "average": 0.0, "closure_args_max": 0, "closure_args_min": 0, "function_args_max": 0, "function_args_min": 0}, "nexits": {"sum": 0, "average": 0.0,"min":0,"max":0}, - "halstead": {"bugs": 0.000_942_552_557_372_941_4, - "difficulty": 1.0, - "effort": 4.754_887_502_163_468, - "length": 3, - "estimated_program_length": 2.0, - "purity_ratio": 0.666_666_666_666_666_6, - "level": 1.0, + "halstead": {"bugs": 0.002_712_967_490_108_627, + "difficulty": 2.0, + "effort": 23.219_280_948_873_62, + "length": 5, + "estimated_program_length": 8.0, + "purity_ratio": 1.6, + "level": 0.5, "total_operands": 1, - "total_operators": 2, - "vocabulary": 3, - "time": 0.264_160_416_786_859_36, + "total_operators": 4, + "vocabulary": 5, + "time": 1.289_960_052_715_201, "unique_operands": 1, - "unique_operators": 2, - "volume": 4.754_887_502_163_468}, + "unique_operators": 4, + "volume": 11.609_640_474_436_81}, "loc": {"cloc": 0, "ploc": 2, "lloc": 1, "sloc": 2, "blank": 0, "cloc_average": 0.0, "ploc_average": 2.0, "lloc_average": 1.0, "sloc_average": 2.0, "blank_average": 0.0, "cloc_min": 0, "ploc_min": 2, "lloc_min": 1, "sloc_min": 2, "blank_min": 0, "cloc_max": 0, "ploc_max": 2, "lloc_max": 1, "sloc_max": 2, "blank_max": 0}, "nom": {"functions": 1, "closures": 0, "functions_average": 1.0, "closures_average": 0.0, "total": 1, "average": 1.0, "closures_min": 0, "closures_max": 0, "functions_min": 1, "functions_max": 1}, "tokens": {"tokens": 6, "average": 6.0, "min": 6, "max": 6}, - "mi": {"original": 151.433_315_883_223_23, - "sei": 142.873_061_717_489_78, - "visual_studio": 88.557_494_668_551_6}, + "mi": {"original": 146.791_469_367_545_94, + "sei": 136.176_292_768_754_45, + "visual_studio": 85.842_964_542_424_53}, "abc": {"assignments": 0, "branches": 0, "conditions": 0, "magnitude": 0.0, "value": 0.0, "assignments_average": 0.0, "branches_average": 0.0, "conditions_average": 0.0, "assignments_min": 0, "assignments_max": 0, "branches_min": 0, "branches_max": 0, "conditions_min": 0, "conditions_max": 0}}, "name": "foo", "spaces": []}]} @@ -836,26 +836,26 @@ async fn test_web_metrics_json_unit() { "cognitive": {"sum": 0, "value": 0, "average": 0.0 , "min":0, "max":0}, "nargs": {"function_args": 0, "function_args_average": 0.0, "closure_args": 0, "closure_args_average": 0.0, "total": 0, "value": 0, "average": 0.0, "closure_args_max": 0, "closure_args_min": 0, "function_args_max": 0, "function_args_min": 0}, "nexits": {"sum": 0, "average": 0.0,"min":0,"max":0}, - "halstead": {"bugs": 0.000_942_552_557_372_941_4, - "difficulty": 1.0, - "effort": 4.754_887_502_163_468, - "length": 3, - "estimated_program_length": 2.0, - "purity_ratio": 0.666_666_666_666_666_6, - "level": 1.0, + "halstead": {"bugs": 0.002_712_967_490_108_627, + "difficulty": 2.0, + "effort": 23.219_280_948_873_62, + "length": 5, + "estimated_program_length": 8.0, + "purity_ratio": 1.6, + "level": 0.5, "total_operands": 1, - "total_operators": 2, - "vocabulary": 3, - "time": 0.264_160_416_786_859_36, + "total_operators": 4, + "vocabulary": 5, + "time": 1.289_960_052_715_201, "unique_operands": 1, - "unique_operators": 2, - "volume": 4.754_887_502_163_468}, + "unique_operators": 4, + "volume": 11.609_640_474_436_81}, "loc": {"cloc": 0, "ploc": 2, "lloc": 1, "sloc": 2, "blank": 0, "cloc_average": 0.0, "ploc_average": 1.0, "lloc_average": 0.5, "sloc_average": 1.0, "blank_average": 0.0, "cloc_min": 0, "ploc_min": 2, "lloc_min": 1, "sloc_min": 2, "blank_min": 0, "cloc_max": 0, "ploc_max": 2, "lloc_max": 1, "sloc_max": 2, "blank_max": 0}, "nom": {"functions": 1, "closures": 0, "functions_average": 0.5, "closures_average": 0.0, "total": 1, "average": 0.5, "closures_min": 0, "closures_max": 0, "functions_min": 0, "functions_max": 1}, "tokens": {"tokens": 6, "average": 3.0, "min": 0, "max": 6}, - "mi": {"original": 151.203_315_883_223_2, - "sei": 142.643_061_717_489_76, - "visual_studio": 88.422_991_744_574_97}, + "mi": {"original": 146.561_469_367_545_93, + "sei": 135.946_292_768_754_43, + "visual_studio": 85.708_461_618_447_91}, "abc": {"assignments": 0, "branches": 0, "conditions": 0, "magnitude": 0.0, "value": 0.0, "assignments_average": 0.0, "branches_average": 0.0, "conditions_average": 0.0, "assignments_min": 0, "assignments_max": 0, "branches_min": 0, "branches_max": 0, "conditions_min": 0, "conditions_max": 0}, "wmc": {"class_wmc_sum": 0, "interface_wmc_sum": 0, "total": 0}, "npm": {"class_npm_sum": 0, "interface_npm_sum": 0, "class_methods": 0, "interface_methods": 0, "class_coa": 0.0, "interface_coa": 0.0, "total": 0, "total_methods": 0, "coa": 0.0}, @@ -942,26 +942,26 @@ async fn test_web_metrics_plain() { "cognitive": {"sum": 0, "value": 0, "average": 0.0 , "min":0, "max":0}, "nargs": {"function_args": 0, "function_args_average": 0.0, "closure_args": 0, "closure_args_average": 0.0, "total": 0, "value": 0, "average": 0.0, "closure_args_max": 0, "closure_args_min": 0, "function_args_max": 0, "function_args_min": 0}, "nexits": {"sum": 0, "average": 0.0,"min":0,"max":0}, - "halstead": {"bugs": 0.000_942_552_557_372_941_4, - "difficulty": 1.0, - "effort": 4.754_887_502_163_468, - "length": 3, - "estimated_program_length": 2.0, - "purity_ratio": 0.666_666_666_666_666_6, - "level": 1.0, + "halstead": {"bugs": 0.002_712_967_490_108_627, + "difficulty": 2.0, + "effort": 23.219_280_948_873_62, + "length": 5, + "estimated_program_length": 8.0, + "purity_ratio": 1.6, + "level": 0.5, "total_operands": 1, - "total_operators": 2, - "vocabulary": 3, - "time": 0.264_160_416_786_859_36, + "total_operators": 4, + "vocabulary": 5, + "time": 1.289_960_052_715_201, "unique_operands": 1, - "unique_operators": 2, - "volume": 4.754_887_502_163_468}, + "unique_operators": 4, + "volume": 11.609_640_474_436_81}, "loc": {"cloc": 0, "ploc": 2, "lloc": 1, "sloc": 2, "blank": 0, "cloc_average": 0.0, "ploc_average": 1.0, "lloc_average": 0.5, "sloc_average": 1.0, "blank_average": 0.0, "cloc_min": 0, "ploc_min": 2, "lloc_min": 1, "sloc_min": 2, "blank_min": 0, "cloc_max": 0, "ploc_max": 2, "lloc_max": 1, "sloc_max": 2, "blank_max": 0}, "nom": {"functions": 1, "closures": 0, "functions_average": 0.5, "closures_average": 0.0, "total": 1, "average": 0.5, "closures_min": 0, "closures_max": 0, "functions_min": 0, "functions_max": 1}, "tokens": {"tokens": 6, "average": 3.0, "min": 0, "max": 6}, - "mi": {"original": 151.203_315_883_223_2, - "sei": 142.643_061_717_489_76, - "visual_studio": 88.422_991_744_574_97}, + "mi": {"original": 146.561_469_367_545_93, + "sei": 135.946_292_768_754_43, + "visual_studio": 85.708_461_618_447_91}, "abc": {"assignments": 0, "branches": 0, "conditions": 0, "magnitude": 0.0, "value": 0.0, "assignments_average": 0.0, "branches_average": 0.0, "conditions_average": 0.0, "assignments_min": 0, "assignments_max": 0, "branches_min": 0, "branches_max": 0, "conditions_min": 0, "conditions_max": 0}, "wmc": {"class_wmc_sum": 0, "interface_wmc_sum": 0, "total": 0}, "npm": {"class_npm_sum": 0, "interface_npm_sum": 0, "class_methods": 0, "interface_methods": 0, "class_coa": 0.0, "interface_coa": 0.0, "total": 0, "total_methods": 0, "coa": 0.0}, @@ -974,26 +974,26 @@ async fn test_web_metrics_plain() { "cognitive": {"sum": 0, "value": 0, "average": 0.0 , "min":0, "max":0}, "nargs": {"function_args": 0, "function_args_average": 0.0, "closure_args": 0, "closure_args_average": 0.0, "total": 0, "value": 0, "average": 0.0, "closure_args_max": 0, "closure_args_min": 0, "function_args_max": 0, "function_args_min": 0}, "nexits": {"sum": 0, "average": 0.0,"min":0,"max":0}, - "halstead": {"bugs": 0.000_942_552_557_372_941_4, - "difficulty": 1.0, - "effort": 4.754_887_502_163_468, - "length": 3, - "estimated_program_length": 2.0, - "purity_ratio": 0.666_666_666_666_666_6, - "level": 1.0, + "halstead": {"bugs": 0.002_712_967_490_108_627, + "difficulty": 2.0, + "effort": 23.219_280_948_873_62, + "length": 5, + "estimated_program_length": 8.0, + "purity_ratio": 1.6, + "level": 0.5, "total_operands": 1, - "total_operators": 2, - "vocabulary": 3, - "time": 0.264_160_416_786_859_36, + "total_operators": 4, + "vocabulary": 5, + "time": 1.289_960_052_715_201, "unique_operands": 1, - "unique_operators": 2, - "volume": 4.754_887_502_163_468}, + "unique_operators": 4, + "volume": 11.609_640_474_436_81}, "loc": {"cloc": 0, "ploc": 2, "lloc": 1, "sloc": 2, "blank": 0, "cloc_average": 0.0, "ploc_average": 2.0, "lloc_average": 1.0, "sloc_average": 2.0, "blank_average": 0.0, "cloc_min": 0, "ploc_min": 2, "lloc_min": 1, "sloc_min": 2, "blank_min": 0, "cloc_max": 0, "ploc_max": 2, "lloc_max": 1, "sloc_max": 2, "blank_max": 0}, "nom": {"functions": 1, "closures": 0, "functions_average": 1.0, "closures_average": 0.0, "total": 1, "average": 1.0, "closures_min": 0, "closures_max": 0, "functions_min": 1, "functions_max": 1}, "tokens": {"tokens": 6, "average": 6.0, "min": 6, "max": 6}, - "mi": {"original": 151.433_315_883_223_23, - "sei": 142.873_061_717_489_78, - "visual_studio": 88.557_494_668_551_6}, + "mi": {"original": 146.791_469_367_545_94, + "sei": 136.176_292_768_754_45, + "visual_studio": 85.842_964_542_424_53}, "abc": {"assignments": 0, "branches": 0, "conditions": 0, "magnitude": 0.0, "value": 0.0, "assignments_average": 0.0, "branches_average": 0.0, "conditions_average": 0.0, "assignments_min": 0, "assignments_max": 0, "branches_min": 0, "branches_max": 0, "conditions_min": 0, "conditions_max": 0}}, "name": "foo", "spaces": []}]} diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 58267b7a..934870b8 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -809,28 +809,31 @@ mod tests { c = 3 + 3", "foo.py", |metric| { - // unique operators: def, =, + - // operators: def, def, def, =, =, =, +, +, + + // unique operators: def, (), :, =, + + // operators: def, (), : for each of the three headers, + // then =, =, =, +, +, + (the `()` and `:` since #1486) // unique operands: foo, bar, toto, a, b, c, 1, 2, 3 // operands: foo, bar, toto, a, b, c, 1, 1, 2, a, 3, 3 + assert_eq!(metric.halstead.unique_operators(), 5); + assert_eq!(metric.halstead.total_operators(), 15); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 3, - "total_operators": 9, + "unique_operators": 5, + "total_operators": 15, "unique_operands": 9, "total_operands": 12, - "length": 21, - "estimated_program_length": 33.284212515144276, - "purity_ratio": 1.584962500721156, - "vocabulary": 12, - "volume": 75.28421251514428, - "difficulty": 2.0, - "level": 0.5, - "effort": 150.56842503028855, - "time": 8.364912501682698, - "bugs": 0.0094341190071077 + "length": 27, + "estimated_program_length": 40.13896548741762, + "purity_ratio": 1.4866283513858378, + "vocabulary": 14, + "volume": 102.79858289555531, + "difficulty": 3.3333333333333335, + "level": 0.3, + "effort": 342.6619429851844, + "time": 19.03677461028802, + "bugs": 0.01632259960095138 } "# ); @@ -1272,9 +1275,12 @@ mod tests { // operand counts pin the rest of the Halstead state. // Grew from 30 → 33 with the issue #394 fix: `const`, // `type`, and `struct` keywords are now classified as - // operators (one occurrence each). - assert_eq!(metric.halstead.unique_operators(), 33); - assert_eq!(metric.halstead.total_operators(), 121); + // operators (one occurrence each). #1486 adds the + // annotation `:` — one entry, 23 occurrences: `C`, the + // two fields, `p`, the four params of `f`, and the + // fifteen `let` bindings. + assert_eq!(metric.halstead.unique_operators(), 34); + assert_eq!(metric.halstead.total_operators(), 144); // u_operands / operands grew (was 31/50 before #390): the // fix now classifies TypeIdentifier (`T`, `S`, `Option`) // and FieldIdentifier (struct fields `x`, `y`) as operands @@ -1293,7 +1299,7 @@ mod tests { // `TokenRole::Unknown`, so the field names were not counted // as operands. Both C++ and Go already classify FieldIdentifier // as an operand. After the fix: - // unique operators: fn, (), {}, let, =, +, ;, . + // unique operators: fn, (), {}, let, =, +, ;, ., ,, : // unique operands : main, p, Point, x, y, sum, 0, 1 // Field names `x` and `y` each appear twice (`p.x + p.y` and // the struct literal `Point { x: 0, y: 1 }`). @@ -1310,24 +1316,28 @@ mod tests { // fix, +Point, +x, +y → 8 distinct names. assert_eq!(metric.halstead.unique_operands(), 8); assert_eq!(metric.halstead.total_operands(), 12); + // N1: fn, (), {} ×2, let ×2, = ×2, +, ; ×2, . ×2, `,`, + // and the struct literal's two field `:` (#1486) = 16. + assert_eq!(metric.halstead.unique_operators(), 10); + assert_eq!(metric.halstead.total_operators(), 16); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 9, - "total_operators": 14, + "unique_operators": 10, + "total_operators": 16, "unique_operands": 8, "total_operands": 12, - "length": 26, - "estimated_program_length": 52.529325012980806, - "purity_ratio": 2.0203586543454155, - "vocabulary": 17, - "volume": 106.27403387250882, - "difficulty": 6.75, - "level": 0.14814814814814814, - "effort": 717.3497286394346, - "time": 39.85276270219081, - "bugs": 0.026711567292222575 + "length": 28, + "estimated_program_length": 57.219280948873624, + "purity_ratio": 2.043545748174058, + "vocabulary": 18, + "volume": 116.75790004038474, + "difficulty": 7.5, + "level": 0.13333333333333333, + "effort": 875.6842503028855, + "time": 48.64912501682697, + "bugs": 0.03051010983631521 } "# ); @@ -1347,7 +1357,8 @@ mod tests { // // Also covers issue #394: `::` is now an operator. The snippet // has two `::` tokens (`Vec::new`, `HashMap::new`), so n1 grew - // from 10 → 11 and N1 from 17 → 19. + // from 10 → 11 and N1 from 17 → 19. #1486 then billed the two + // annotation `:` tokens as their own entry: 12 and 21. check_metrics::( "fn main() { let v: Vec = Vec::new(); @@ -1362,26 +1373,27 @@ mod tests { assert_eq!(metric.halstead.total_operands(), 11); // `::` appears twice (Vec::new, HashMap::new); without // the #394 fix u_operators was 10 and operators 17. - assert_eq!(metric.halstead.unique_operators(), 11); - assert_eq!(metric.halstead.total_operators(), 19); + // `:` appears twice and stays distinct from `::`. + assert_eq!(metric.halstead.unique_operators(), 12); + assert_eq!(metric.halstead.total_operators(), 21); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 11, - "total_operators": 19, + "unique_operators": 12, + "total_operators": 21, "unique_operands": 8, "total_operands": 11, - "length": 30, - "estimated_program_length": 62.05374780501027, - "purity_ratio": 2.068458260167009, - "vocabulary": 19, - "volume": 127.43782540330756, - "difficulty": 7.5625, - "level": 0.1322314049586777, - "effort": 963.7485546125134, - "time": 53.54158636736186, - "bugs": 0.03252279825177962 + "length": 32, + "estimated_program_length": 67.01955000865388, + "purity_ratio": 2.0943609377704338, + "vocabulary": 20, + "volume": 138.3016990363956, + "difficulty": 8.25, + "level": 0.12121212121212122, + "effort": 1140.9890170502638, + "time": 63.38827872501466, + "bugs": 0.03639706698498207 } "# ); @@ -1389,6 +1401,45 @@ mod tests { ); } + /// #1486: Rust's single `:` is vocabulary in every position it is + /// spelled, and stays a separate entry from `::`. + #[cfg(feature = "rust")] + #[test] + fn rust_colon_is_operator_1486() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `let`, `:`, `u8`, `=`, `;` + ("let annotation", "let x: u8 = 1;\n", [5, 5]), + // `struct`, `{}`, `:`, `u8` + ("field declaration", "struct S { a: u8 }\n", [4, 4]), + // `const`, `:` twice, `=`, `{}`, `;` + ("field initializer", "const C: S = S { a: 1 };\n", [5, 6]), + // `fn`, `{}` twice, `()`, `let`, `{}` (pattern), `:`, `=`, `;` + ("field pattern", "fn f() { let S { a: b } = s; }\n", [7, 8]), + // `fn`, `()`, `:`, `u8`, `{}` + ("parameter", "fn f(x: u8) {}\n", [5, 5]), + // `fn`, `<`, `:`, `>`, `()`, `{}` + ("trait bound", "fn f() {}\n", [6, 6]), + // `fn`, `<`, `>`, `()`, `where`, `:`, `{}` + ("where clause", "fn f() where T: Clone {}\n", [7, 7]), + // `fn`, `()`, `{}` twice, `:`, `loop`, `break`, `;` + ("loop label", "fn f() { 'a: loop { break 'a; } }\n", [7, 8]), + // `fn`, `()`, `{}`, `let`, `=`, `|` twice, `:`, `u8`, `;` + ( + "closure parameter", + "fn f() { let c = |q: u8| q; }\n", + [9, 10], + ), + // `fn`, `()` twice, `{}`, `let`, `:`, `<`, `u8`, `>`, `=`, + // `::`, `;` — `:` and `::` are two entries + ( + "beside a path", + "fn f() { let v: Vec = Vec::new(); }\n", + [11, 12], + ), + ]; + assert_operator_rows::("foo.rs", ROWS); + } + #[cfg(feature = "rust")] #[test] fn rust_path_separator_is_operator() { @@ -2546,20 +2597,26 @@ mod tests { #[cfg(feature = "python")] #[test] fn python_wrong_operators() { + // Not valid Python — the parser recovers `()` as a tuple, the `[` + // as a bare token and `{}` as a dict — but each opener is still a + // delimiter the source spells, so since #1486 the three are + // billed once each as `()`, `[]` and `{}`. The closers never are. check_metrics::("()[]{}", "foo.py", |metric| { + assert_eq!(metric.halstead.unique_operators(), 3); + assert_eq!(metric.halstead.total_operators(), 3); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 0, - "total_operators": 0, + "unique_operators": 3, + "total_operators": 3, "unique_operands": 0, "total_operands": 0, - "length": 0, - "estimated_program_length": 0.0, - "purity_ratio": 0.0, - "vocabulary": 0, - "volume": 0.0, + "length": 3, + "estimated_program_length": 4.754887502163468, + "purity_ratio": 1.584962500721156, + "vocabulary": 3, + "volume": 4.754887502163468, "difficulty": 0.0, "level": 0.0, "effort": 0.0, @@ -2579,24 +2636,28 @@ mod tests { pass", "foo.py", |metric| { + // expected operators: def, (), :, pass — one each (#1486 + // added the `()` and the block `:`). + assert_eq!(metric.halstead.unique_operators(), 4); + assert_eq!(metric.halstead.total_operators(), 4); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 2, - "total_operators": 2, + "unique_operators": 4, + "total_operators": 4, "unique_operands": 1, "total_operands": 1, - "length": 3, - "estimated_program_length": 2.0, - "purity_ratio": 0.6666666666666666, - "vocabulary": 3, - "volume": 4.754887502163468, - "difficulty": 1.0, - "level": 1.0, - "effort": 4.754887502163468, - "time": 0.26416041678685936, - "bugs": 0.0009425525573729414 + "length": 5, + "estimated_program_length": 8.0, + "purity_ratio": 1.6, + "vocabulary": 5, + "volume": 11.60964047443681, + "difficulty": 2.0, + "level": 0.5, + "effort": 23.21928094887362, + "time": 1.289960052715201, + "bugs": 0.002712967490108627 } "# ); @@ -5718,12 +5779,13 @@ end", "async def f():\n await a()\n await b()\n await c()\n", "foo.py", |metric| { - // expected operators: async, def, await (3 unique) - // await used three times -> N1 counts: async(1) def(1) await(3) = 5 + // expected operators: async, def, (), :, await (5 unique) + // N1: async(1) def(1) await(3) = 5, plus the four call + // and parameter-list `(` and the block `:` (#1486) = 10. // Before #413, Await + Await2 both matched, so `await` was a - // distinct operator twice: n1=4, N1=8. - assert_eq!(metric.halstead.unique_operators(), 3); - assert_eq!(metric.halstead.total_operators(), 5); + // distinct operator twice: one more unique and three more total. + assert_eq!(metric.halstead.unique_operators(), 5); + assert_eq!(metric.halstead.total_operators(), 10); }, ); } @@ -5735,10 +5797,11 @@ end", #[test] fn python_lambda_counted_once() { check_metrics::("g = lambda x: x + 1\n", "foo.py", |metric| { - // expected operators: =, lambda, + (3 unique, each used once) + // expected operators: =, lambda, :, + (4 unique, each used + // once; the parameter-list `:` since #1486). // Before #413, lambda was absent: only =, + were counted. - assert_eq!(metric.halstead.unique_operators(), 3); - assert_eq!(metric.halstead.total_operators(), 3); + assert_eq!(metric.halstead.unique_operators(), 4); + assert_eq!(metric.halstead.total_operators(), 4); }); } @@ -5752,11 +5815,12 @@ end", "match x:\n case 1:\n pass\n case _:\n pass\n", "foo.py", |metric| { - // expected operators: match, case, pass (3 unique) - // match(1) + case(2) + pass(2) = 5 total occurrences. + // expected operators: match, case, pass, : (4 unique) + // match(1) + case(2) + pass(2) + one block `:` per + // header (3, #1486) = 8 total occurrences. // Before #413, neither match nor case was counted (only pass). - assert_eq!(metric.halstead.unique_operators(), 3); - assert_eq!(metric.halstead.total_operators(), 5); + assert_eq!(metric.halstead.unique_operators(), 4); + assert_eq!(metric.halstead.total_operators(), 8); }, ); } @@ -5770,10 +5834,11 @@ end", "def f():\n global a\n nonlocal b\n", "foo.py", |metric| { - // expected operators: def, global, nonlocal (3 unique) + // expected operators: def, (), :, global, nonlocal + // (5 unique, each once; `()` and `:` since #1486) // Before #413, nonlocal was absent: only def, global counted. - assert_eq!(metric.halstead.unique_operators(), 3); - assert_eq!(metric.halstead.total_operators(), 3); + assert_eq!(metric.halstead.unique_operators(), 5); + assert_eq!(metric.halstead.total_operators(), 5); }, ); } @@ -5789,22 +5854,119 @@ end", "a not in b\na is not b\nnot c\nd in e\nf is g\nfor h in i:\n pass\n", "foo.py", |metric| { - // expected operators (7 unique): + // expected operators (8 unique): // "not in" (compound, once), "is not" (compound, once), // "not" (standalone `not c`, once), // "in" (standalone `d in e` + `for h in i` = twice), // "is" (standalone `f is g`, once), - // "for" (once), "pass" (once) - // Total occurrences: 1+1+1+2+1+1+1 = 8. + // "for" (once), "pass" (once), and the `for` header's + // block ":" (once, #1486) + // Total occurrences: 1+1+1+2+1+1+1+1 = 9. // Before #413, `a not in b` counted not+in (two) and // `a is not b` counted is+not (two); the compounds were // never classified. - assert_eq!(metric.halstead.unique_operators(), 7); - assert_eq!(metric.halstead.total_operators(), 8); + assert_eq!(metric.halstead.unique_operators(), 8); + assert_eq!(metric.halstead.total_operators(), 9); }, ); } + /// Asserts `[n1, N1]` for each `(label, source, expected)` row, one + /// parse per row so a row's counts are its own. + #[cfg(any(feature = "python", feature = "rust"))] + fn assert_operator_rows(file: &str, rows: &[(&str, &str, [u64; 2])]) { + for (label, source, expected) in rows { + crate::test_support::check_func_space_only::( + source, + file, + &[crate::Metric::Halstead], + |space| { + let halstead = &space.metrics.halstead; + assert_eq!( + [halstead.unique_operators(), halstead.total_operators()], + *expected, + "{label}: {source:?}" + ); + }, + ); + } + } + + /// #1486: Python's bracket openers, `;` and `:` are vocabulary, as + /// they are in every other getter (#1395). Each row isolates one + /// position, so the expected `[n1, N1]` is derivable from the row + /// alone; a bracket pair bills its opener once, as `()` / `[]` / `{}`. + #[cfg(feature = "python")] + #[test] + fn python_punctuation_is_vocabulary_1486() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `()` + ("call", "f(x)\n", [1, 1]), + // `()`, `=` + ("keyword argument", "f(a=1)\n", [2, 2]), + // `()`, `*`, `,`, `**` + ("splat arguments", "f(*a, **k)\n", [4, 4]), + // `[]` + ("index", "a[1]\n", [1, 1]), + // `[]`, `:` twice + ("slice", "a[1:2:3]\n", [2, 3]), + // `{}`, `:` + ("dict", "{k: v}\n", [2, 2]), + // `{}`, `,` + ("set", "{1, 2}\n", [2, 2]), + // `[]`, `,` twice, `()` + ("list and tuple", "[1, (2, 3)]\n", [3, 4]), + // `()`, `for`, `in` + ("generator", "(x for x in y)\n", [3, 3]), + // `{}`, `:`, `for`, `in` + ("dict comprehension", "{k: v for k in y}\n", [4, 4]), + // `()`, `:=` — the walrus is its own token, not a `:` + ("walrus", "(n := 3)\n", [2, 2]), + // `lambda`, `:` + ("lambda", "lambda x: x\n", [2, 2]), + // `def`, `()`, `:` twice, `->`, `pass` + ("annotations", "def f(x: int) -> int:\n pass\n", [5, 6]), + // `:`, `=` + ("annotated assignment", "x: int = 1\n", [2, 2]), + // `@`, `def`, `()`, `:`, `pass` + ("decorator", "@d\ndef f():\n pass\n", [5, 5]), + // `if`, `:`, `pass` + ("if block", "if x:\n pass\n", [3, 3]), + // `while`, `:`, `pass` + ("while block", "while x:\n pass\n", [3, 3]), + // `for`, `in`, `:`, `pass` + ("for block", "for i in y:\n pass\n", [4, 4]), + // `with`, `:`, `pass` + ("with block", "with o:\n pass\n", [3, 3]), + // `try`, `except`, `:` twice, `pass` twice + ("try block", "try:\n pass\nexcept E:\n pass\n", [4, 6]), + // `:`, `pass` — the `class` keyword itself is not billed + ("class block", "class C:\n pass\n", [2, 2]), + // `;` + ("statement separator", "a; b\n", [1, 1]), + ]; + assert_operator_rows::("foo.py", ROWS); + } + + /// #1486: an f-string replacement field's `{` and format-spec `:` are + /// spelling, not operations, like every other language's + /// interpolation opener (#1314) — the f-string rows bill nothing. + /// The guard reads the token's *parent*, so a dict written inside a + /// replacement field keeps its own `{}` and `:`. + #[cfg(feature = "python")] + #[test] + fn python_f_string_delimiters_are_not_operators_1486() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + ("interpolation opener", "f\"{x}\"\n", [0, 0]), + ("format spec colon", "f\"{x:>10}\"\n", [0, 0]), + ("nested format expression", "f\"{a:{w}}\"\n", [0, 0]), + // `{}`, `:`, `[]` of the dict and its subscript, nothing of + // the replacement field around them + ("dict inside a field", "f\"{ {k: v}[k] }\"\n", [3, 3]), + ]; + assert_operator_rows::("foo.py", ROWS); + } + #[cfg(feature = "bash")] #[test] fn bash_operators_and_operands() { diff --git a/src/metrics/mi.rs b/src/metrics/mi.rs index 67f26815..85b2b104 100644 --- a/src/metrics/mi.rs +++ b/src/metrics/mi.rs @@ -225,9 +225,9 @@ mod tests { metric.mi, @r#" { - "original": 151.2033158832232, - "sei": 142.64306171748976, - "visual_studio": 88.42299174457497 + "original": 146.56146936754593, + "sei": 135.94629276875443, + "visual_studio": 85.70846161844791 } "# ); diff --git a/src/ops.rs b/src/ops.rs index b3583772..599ba938 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -461,7 +461,7 @@ mod tests { "if True: a = 1 + 2", "foo.py", - &mut ["if", "=", "+"], + &mut ["if", ":", "=", "+"], &mut ["True", "a", "1", "2"], ); } @@ -507,11 +507,27 @@ mod tests { b = 2 + a c = 3 + 3", "foo.py", - &mut ["def", "=", "+"], + &mut ["def", "()", ":", "=", "+"], &mut ["foo", "bar", "toto", "a", "b", "c", "1", "2", "3"], ); } + /// #1486: Python bills a call's parentheses the way the JS family + /// does — one `()` entry for the pair, rendered as the pair. Before + /// it, Python reported no operator here at all. + #[cfg(all(feature = "javascript", feature = "python"))] + #[test] + fn python_call_parens_pair_like_javascript() { + check_ops(LANG::Python, "f(x)", "foo.py", &mut ["()"], &mut ["f", "x"]); + check_ops( + LANG::Javascript, + "f(x)", + "foo.js", + &mut ["()"], + &mut ["f", "x"], + ); + } + #[cfg(feature = "cpp")] #[test] fn cpp_ops() { @@ -562,7 +578,7 @@ mod tests { LANG::Rust, "let: usize a = 5; let b: f32 = 7.0; let c: i32 = 3;", "foo.rs", - &mut ["let", "usize", "=", ";", "f32", "i32"], + &mut ["let", ":", "usize", "=", ";", "f32", "i32"], &mut ["a", "b", "c", "5", "7.0", "3"], ); } diff --git a/tests/output_formats/snapshots/output_formats__csv_test__csv_python_greeter.snap b/tests/output_formats/snapshots/output_formats__csv_test__csv_python_greeter.snap index 264489d1..7b7f746b 100644 --- a/tests/output_formats/snapshots/output_formats__csv_test__csv_python_greeter.snap +++ b/tests/output_formats/snapshots/output_formats__csv_test__csv_python_greeter.snap @@ -3,7 +3,7 @@ source: tests/output_formats/csv_test.rs expression: out --- path,space_name,space_kind,start_line,end_line,cognitive.sum,cognitive.average,cognitive.min,cognitive.max,cyclomatic.sum,cyclomatic.average,cyclomatic.min,cyclomatic.max,cyclomatic.modified.sum,cyclomatic.modified.average,cyclomatic.modified.min,cyclomatic.modified.max,halstead.unique_operators,halstead.total_operators,halstead.unique_operands,halstead.total_operands,halstead.length,halstead.estimated_program_length,halstead.purity_ratio,halstead.vocabulary,halstead.volume,halstead.difficulty,halstead.level,halstead.effort,halstead.time,halstead.bugs,loc.sloc,loc.ploc,loc.lloc,loc.cloc,loc.blank,loc.sloc_average,loc.ploc_average,loc.lloc_average,loc.cloc_average,loc.blank_average,loc.sloc_min,loc.sloc_max,loc.cloc_min,loc.cloc_max,loc.ploc_min,loc.ploc_max,loc.lloc_min,loc.lloc_max,loc.blank_min,loc.blank_max,nom.functions,nom.closures,nom.functions_average,nom.closures_average,nom.total,nom.average,nom.functions_min,nom.functions_max,nom.closures_min,nom.closures_max,nargs.function_args,nargs.closure_args,nargs.function_args_average,nargs.closure_args_average,nargs.total,nargs.average,nargs.function_args_min,nargs.function_args_max,nargs.closure_args_min,nargs.closure_args_max,nexits.sum,nexits.average,nexits.min,nexits.max,tokens.sum,tokens.average,tokens.min,tokens.max,abc.assignments,abc.branches,abc.conditions,abc.magnitude,abc.assignments_average,abc.branches_average,abc.conditions_average,abc.assignments_min,abc.assignments_max,abc.branches_min,abc.branches_max,abc.conditions_min,abc.conditions_max,wmc.class_wmc_sum,wmc.interface_wmc_sum,wmc.total,npm.class_npm_sum,npm.interface_npm_sum,npm.class_methods,npm.interface_methods,npm.class_coa,npm.interface_coa,npm.total,npm.total_methods,npm.coa,npa.class_npa_sum,npa.interface_npa_sum,npa.class_attributes,npa.interface_attributes,npa.class_cda,npa.interface_cda,npa.total,npa.total_attributes,npa.cda,mi.original,mi.sei,mi.visual_studio -greeter.py,greeter.py,unit,1,9,1,0.5,0,1,5,2.5,1,2,5,2.5,1,2,6,10,6,14,24,31.019550008653873,1.292481250360578,12,86.03910001730775,7,0.14285714285714285,602.2737001211542,33.45965000673079,0.02377249024853662,9,7,4,0,2,2.25,1.75,1,0,0.5,2,9,0,0,2,7,1,4,0,2,2,0,0.5,0,2,0.5,0,1,0,0,3,0,1.5,0,3,1.5,0,2,0,0,2,1,0,2,41,10.25,0,25,1,0,1,1.4142135623730951,0.25,0,0.25,0,1,0,0,0,1,3,0,3,2,0,2,0,1,0,2,2,1,1,0,1,0,1,0,1,1,1,111.08999225681957,85.0772282263165,64.96490775252606 -greeter.py,Greeter,class,2,9,1,0.5,0,1,4,2,1,2,4,2,1,2,6,10,6,14,24,31.019550008653873,1.292481250360578,12,86.03910001730775,7,0.14285714285714285,602.2737001211542,33.45965000673079,0.02377249024853662,8,7,4,0,1,2.6666666666666665,2.3333333333333335,1.3333333333333333,0,0.3333333333333333,2,8,0,0,2,7,1,4,0,1,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,3,0,1.5,0,3,1.5,0,2,0,0,2,1,0,2,41,13.666666666666666,3,25,1,0,1,1.4142135623730951,0.3333333333333333,0,0.3333333333333333,0,1,0,0,0,1,3,0,3,2,0,2,0,1,0,2,2,1,1,0,1,0,1,0,1,1,1,113.22807743445301,88.06001324968199,66.21524996166842 -greeter.py,__init__,function,3,4,0,0,0,0,1,1,1,1,1,1,1,1,4,4,3,6,10,12.754887502163468,1.2754887502163468,7,28.07354922057604,4,0.25,112.29419688230416,6.2385664934613425,0.007758604623869012,2,2,1,0,0,2,2,1,0,0,2,2,0,0,2,2,1,1,0,0,1,0,1,0,1,1,1,1,0,0,2,0,2,0,2,2,2,2,0,0,0,0,0,0,13,13,13,13,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,142.19991098932817,129.5520742665475,83.15784268381765 -greeter.py,greet,function,6,9,1,1,1,1,2,2,2,2,2,2,2,2,4,6,4,7,13,16,1.2307692307692308,8,39,3.5,0.2857142857142857,136.5,7.583333333333333,0.008836911647049637,4,4,3,0,0,4,4,3,0,0,4,4,0,0,4,4,3,3,0,0,1,0,1,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,2,2,2,2,25,25,25,25,0,0,1,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,129.0315107899836,110.65590846191628,75.45702385379158 +greeter.py,greeter.py,unit,1,9,1,0.5,0,1,5,2.5,1,2,5,2.5,1,2,8,16,6,14,30,39.50977500432694,1.3169925001442313,14,114.22064766172812,9.333333333333334,0.10714285714285714,1066.0593781761293,59.225521009784956,0.034785614145326625,9,7,4,0,2,2.25,1.75,1,0,0.5,2,9,0,0,2,7,1,4,0,2,2,0,0.5,0,2,0.5,0,1,0,0,3,0,1.5,0,3,1.5,0,2,0,0,2,1,0,2,41,10.25,0,25,1,0,1,1.4142135623730951,0.25,0,0.25,0,1,0,0,0,1,3,0,3,2,0,2,0,1,0,2,2,1,1,0,1,0,1,0,1,1,1,109.61667501284309,82.95168074477543,64.10331872096087 +greeter.py,Greeter,class,2,9,1,0.5,0,1,4,2,1,2,4,2,1,2,8,16,6,14,30,39.50977500432694,1.3169925001442313,14,114.22064766172812,9.333333333333334,0.10714285714285714,1066.0593781761293,59.225521009784956,0.034785614145326625,8,7,4,0,1,2.6666666666666665,2.3333333333333335,1.3333333333333333,0,0.3333333333333333,2,8,0,0,2,7,1,4,0,1,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,3,0,1.5,0,3,1.5,0,2,0,0,2,1,0,2,41,13.666666666666666,3,25,1,0,1,1.4142135623730951,0.3333333333333333,0,0.3333333333333333,0,1,0,0,0,1,3,0,3,2,0,2,0,1,0,2,2,1,1,0,1,0,1,0,1,1,1,111.75476019047653,85.93446576814091,65.35366093010323 +greeter.py,__init__,function,3,4,0,0,0,0,1,1,1,1,1,1,1,1,6,6,3,6,12,20.264662506490403,1.6887218755408668,9,38.039100017307746,6,0.16666666666666666,228.23460010384647,12.679700005769249,0.012448959890820123,2,2,1,0,0,2,2,1,0,0,2,2,0,0,2,2,1,1,0,0,1,0,1,0,1,1,1,1,0,0,2,0,2,0,2,2,2,2,0,0,0,0,0,0,13,13,13,13,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,140.6202198664881,127.27306171748977,82.2340467055486 +greeter.py,greet,function,6,9,1,1,1,1,2,2,2,2,2,2,2,2,6,9,4,7,16,23.509775004326936,1.4693609377704335,10,53.150849518197795,5.25,0.19047619047619047,279.0419599705384,15.502331109474357,0.014233938588537278,4,4,3,0,0,4,4,3,0,0,4,4,0,0,4,4,3,3,0,0,1,0,1,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,2,2,2,2,25,25,25,25,0,0,1,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,127.4217340918969,108.33349160264797,74.51563397186952 diff --git a/tests/output_formats/snapshots/output_formats__csv_test__csv_rust_counter.snap b/tests/output_formats/snapshots/output_formats__csv_test__csv_rust_counter.snap index c655dd4e..619547a2 100644 --- a/tests/output_formats/snapshots/output_formats__csv_test__csv_rust_counter.snap +++ b/tests/output_formats/snapshots/output_formats__csv_test__csv_rust_counter.snap @@ -3,6 +3,6 @@ source: tests/output_formats/csv_test.rs expression: out --- path,space_name,space_kind,start_line,end_line,cognitive.sum,cognitive.average,cognitive.min,cognitive.max,cyclomatic.sum,cyclomatic.average,cyclomatic.min,cyclomatic.max,cyclomatic.modified.sum,cyclomatic.modified.average,cyclomatic.modified.min,cyclomatic.modified.max,halstead.unique_operators,halstead.total_operators,halstead.unique_operands,halstead.total_operands,halstead.length,halstead.estimated_program_length,halstead.purity_ratio,halstead.vocabulary,halstead.volume,halstead.difficulty,halstead.level,halstead.effort,halstead.time,halstead.bugs,loc.sloc,loc.ploc,loc.lloc,loc.cloc,loc.blank,loc.sloc_average,loc.ploc_average,loc.lloc_average,loc.cloc_average,loc.blank_average,loc.sloc_min,loc.sloc_max,loc.cloc_min,loc.cloc_max,loc.ploc_min,loc.ploc_max,loc.lloc_min,loc.lloc_max,loc.blank_min,loc.blank_max,nom.functions,nom.closures,nom.functions_average,nom.closures_average,nom.total,nom.average,nom.functions_min,nom.functions_max,nom.closures_min,nom.closures_max,nargs.function_args,nargs.closure_args,nargs.function_args_average,nargs.closure_args_average,nargs.total,nargs.average,nargs.function_args_min,nargs.function_args_max,nargs.closure_args_min,nargs.closure_args_max,nexits.sum,nexits.average,nexits.min,nexits.max,tokens.sum,tokens.average,tokens.min,tokens.max,abc.assignments,abc.branches,abc.conditions,abc.magnitude,abc.assignments_average,abc.branches_average,abc.conditions_average,abc.assignments_min,abc.assignments_max,abc.branches_min,abc.branches_max,abc.conditions_min,abc.conditions_max,wmc.class_wmc_sum,wmc.interface_wmc_sum,wmc.total,npm.class_npm_sum,npm.interface_npm_sum,npm.class_methods,npm.interface_methods,npm.class_coa,npm.interface_coa,npm.total,npm.total_methods,npm.coa,npa.class_npa_sum,npa.interface_npa_sum,npa.class_attributes,npa.interface_attributes,npa.class_cda,npa.interface_cda,npa.total,npa.total_attributes,npa.cda,mi.original,mi.sei,mi.visual_studio -counter.rs,counter.rs,unit,1,13,2,2,0,2,4,4,1,2,4,4,1,2,15,23,6,15,38,74.11313393845472,1.950345629959335,21,166.9080620655929,18.75,0.05333333333333334,3129.5261637298668,173.86256465165926,0.07131777790263222,13,11,3,0,2,4.333333333333333,3.6666666666666665,1,0,0.6666666666666666,8,13,0,0,8,11,3,3,0,2,1,0,0.3333333333333333,0,1,0.3333333333333333,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,45,15,4,34,1,0,2,2.23606797749979,0.3333333333333333,0,0.6666666666666666,0,1,0,0,0,2,2,0,2,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,1,0,101.91711611092933,71.74174544054752,59.6006526964499 +counter.rs,counter.rs,unit,1,13,2,2,0,2,4,4,1,2,4,4,1,2,16,24,6,15,39,79.50977500432694,2.0387121795981265,22,173.9178331268546,20,0.05,3478.356662537092,193.24203680761622,0.07652349898080879,13,11,3,0,2,4.333333333333333,3.6666666666666665,1,0,0.6666666666666666,8,13,0,0,8,11,3,3,0,2,1,0,0.3333333333333333,0,1,0.3333333333333333,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,45,15,4,34,1,0,2,2.23606797749979,0.3333333333333333,0,0.6666666666666666,0,1,0,0,0,2,2,0,2,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,1,0,101.7031889949032,71.43311385114494,59.47554911982643 counter.rs,Counter,impl,4,13,2,2,0,2,3,3,1,2,3,3,1,2,14,20,6,13,33,68.81274391313339,2.085234664034345,20,142.62362713128297,15.166666666666666,0.06593406593406594,2163.125011491125,120.17361174950695,0.05575277534828428,10,10,3,0,0,5,5,1.5,0,0,8,10,0,0,8,10,3,3,0,0,1,0,0.5,0,1,0.5,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,38,19,4,34,1,0,2,2.23606797749979,0.5,0,1,0,1,0,0,0,2,2,0,2,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,107.21503374494837,79.28320507878054,62.69885014324466 counter.rs,bump,function,5,12,2,2,2,2,2,2,2,2,2,2,2,2,13,18,5,12,30,59.715356810271004,1.9905118936757,18,125.09775004326937,15.6,0.06410256410256411,1951.524900675002,108.41805003750011,0.05205488323759921,8,8,3,0,0,8,8,3,0,0,8,8,0,0,8,8,3,3,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,34,34,34,34,1,0,2,2.23606797749979,1,0,2,1,1,0,0,2,2,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,111.74175077815008,85.71205743468377,65.34605308663747 diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index 92d630d1..6b147c7a 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit 92d630d1c1537bc4edd72e95e2fe7a3f6886fac4 +Subproject commit 6b147c7a2431293ce44fa306b715fa6e1f31e279 From cd61485aebf3c72f96af0e1039ec152bf2df1ce2 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:17:16 -0700 Subject: [PATCH 26/48] fix(getter/python): bill the class keyword Python billed `def` as a Halstead operator but not `class`, so `class C:\n pass` reported only `:` and `pass`. Ruby, Kotlin, C# and PHP bill their `class` keyword, and Rust bills `struct` / `trait`. The `class` leaf joins the operator arm beside `def`. Its `ClassDefinition` wrapper stays unlisted, so a definition bills one `class`, not two. Every Python file with a class gains one operator entry and one occurrence per definition; the CSV greeter snapshot moves by exactly that (n1 8 -> 9, N1 16 -> 17). The getter's own effort grew, so its self-scan baseline is refreshed. Fixes #1550 --- .bca-baseline.toml | 2 +- big-code-analysis-ast/src/getter/python.rs | 6 +++++- big-code-analysis-book/src/metrics.md | 2 ++ src/metrics/halstead.rs | 12 ++++++++++-- ...output_formats__csv_test__csv_python_greeter.snap | 4 ++-- 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 8b89e54a..29dd27d3 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -261,7 +261,7 @@ value = 96592.02057608281 path = "big-code-analysis-ast/src/getter/python.rs" qualified = "PythonCode::get_op_type" metric = "halstead.effort" -value = 64442.20035097394 +value = 64665.475362488345 [[entry]] path = "big-code-analysis-ast/src/getter/ruby.rs" diff --git a/big-code-analysis-ast/src/getter/python.rs b/big-code-analysis-ast/src/getter/python.rs index 7179e7ac..49f41b2b 100644 --- a/big-code-analysis-ast/src/getter/python.rs +++ b/big-code-analysis-ast/src/getter/python.rs @@ -85,7 +85,11 @@ impl Getter for PythonCode { // `lambda`: count only the keyword token (Lambda3=73), not the // Lambda/Lambda2 expression nodes that wrap it, to avoid the same // node+keyword double count fixed for await (#413). - | Lambda3 => { + | Lambda3 + // `class`, the declaration keyword beside `def`. Only the leaf: + // its `ClassDefinition` wrapper is not listed, so the pair + // cannot double-count (#1550). + | Class => { TokenRole::Operator } Identifier | Integer | Float | True | False | None => TokenRole::Operand, diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 44770ed0..c2af115a 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -708,6 +708,8 @@ tokens involved the same way they spell real operators: slice, dict and annotation `:`, and Rust's annotation, field, bound and label `:`, which is a separate operator from `::`. Both languages left these out until #1486, so their Halstead scores rose with it. + Python's `class` keyword followed in #1550: it is now billed like + `def`, as Ruby, Kotlin, C# and PHP bill their `class`. - **An alternative spelling is an operator of its own.** C++ spells `&&` equally as `and`, `!=` as `not_eq` and so on through all eleven [alternative tokens](https://en.cppreference.com/w/cpp/language/operator_alternative). diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 934870b8..dba2ee86 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -5940,8 +5940,16 @@ end", ("with block", "with o:\n pass\n", [3, 3]), // `try`, `except`, `:` twice, `pass` twice ("try block", "try:\n pass\nexcept E:\n pass\n", [4, 6]), - // `:`, `pass` — the `class` keyword itself is not billed - ("class block", "class C:\n pass\n", [2, 2]), + // `class`, `:`, `pass` — the keyword is billed like `def` + // (#1550) + ("class block", "class C:\n pass\n", [3, 3]), + // `class` twice, `()`, `:` twice, `pass` — one `class` per + // keyword, nothing for the `ClassDefinition` wrapper + ( + "nested class", + "class C(B):\n class D:\n pass\n", + [4, 6], + ), // `;` ("statement separator", "a; b\n", [1, 1]), ]; diff --git a/tests/output_formats/snapshots/output_formats__csv_test__csv_python_greeter.snap b/tests/output_formats/snapshots/output_formats__csv_test__csv_python_greeter.snap index 7b7f746b..9ea8f483 100644 --- a/tests/output_formats/snapshots/output_formats__csv_test__csv_python_greeter.snap +++ b/tests/output_formats/snapshots/output_formats__csv_test__csv_python_greeter.snap @@ -3,7 +3,7 @@ source: tests/output_formats/csv_test.rs expression: out --- path,space_name,space_kind,start_line,end_line,cognitive.sum,cognitive.average,cognitive.min,cognitive.max,cyclomatic.sum,cyclomatic.average,cyclomatic.min,cyclomatic.max,cyclomatic.modified.sum,cyclomatic.modified.average,cyclomatic.modified.min,cyclomatic.modified.max,halstead.unique_operators,halstead.total_operators,halstead.unique_operands,halstead.total_operands,halstead.length,halstead.estimated_program_length,halstead.purity_ratio,halstead.vocabulary,halstead.volume,halstead.difficulty,halstead.level,halstead.effort,halstead.time,halstead.bugs,loc.sloc,loc.ploc,loc.lloc,loc.cloc,loc.blank,loc.sloc_average,loc.ploc_average,loc.lloc_average,loc.cloc_average,loc.blank_average,loc.sloc_min,loc.sloc_max,loc.cloc_min,loc.cloc_max,loc.ploc_min,loc.ploc_max,loc.lloc_min,loc.lloc_max,loc.blank_min,loc.blank_max,nom.functions,nom.closures,nom.functions_average,nom.closures_average,nom.total,nom.average,nom.functions_min,nom.functions_max,nom.closures_min,nom.closures_max,nargs.function_args,nargs.closure_args,nargs.function_args_average,nargs.closure_args_average,nargs.total,nargs.average,nargs.function_args_min,nargs.function_args_max,nargs.closure_args_min,nargs.closure_args_max,nexits.sum,nexits.average,nexits.min,nexits.max,tokens.sum,tokens.average,tokens.min,tokens.max,abc.assignments,abc.branches,abc.conditions,abc.magnitude,abc.assignments_average,abc.branches_average,abc.conditions_average,abc.assignments_min,abc.assignments_max,abc.branches_min,abc.branches_max,abc.conditions_min,abc.conditions_max,wmc.class_wmc_sum,wmc.interface_wmc_sum,wmc.total,npm.class_npm_sum,npm.interface_npm_sum,npm.class_methods,npm.interface_methods,npm.class_coa,npm.interface_coa,npm.total,npm.total_methods,npm.coa,npa.class_npa_sum,npa.interface_npa_sum,npa.class_attributes,npa.interface_attributes,npa.class_cda,npa.interface_cda,npa.total,npa.total_attributes,npa.cda,mi.original,mi.sei,mi.visual_studio -greeter.py,greeter.py,unit,1,9,1,0.5,0,1,5,2.5,1,2,5,2.5,1,2,8,16,6,14,30,39.50977500432694,1.3169925001442313,14,114.22064766172812,9.333333333333334,0.10714285714285714,1066.0593781761293,59.225521009784956,0.034785614145326625,9,7,4,0,2,2.25,1.75,1,0,0.5,2,9,0,0,2,7,1,4,0,2,2,0,0.5,0,2,0.5,0,1,0,0,3,0,1.5,0,3,1.5,0,2,0,0,2,1,0,2,41,10.25,0,25,1,0,1,1.4142135623730951,0.25,0,0.25,0,1,0,0,0,1,3,0,3,2,0,2,0,1,0,2,2,1,1,0,1,0,1,0,1,1,1,109.61667501284309,82.95168074477543,64.10331872096087 -greeter.py,Greeter,class,2,9,1,0.5,0,1,4,2,1,2,4,2,1,2,8,16,6,14,30,39.50977500432694,1.3169925001442313,14,114.22064766172812,9.333333333333334,0.10714285714285714,1066.0593781761293,59.225521009784956,0.034785614145326625,8,7,4,0,1,2.6666666666666665,2.3333333333333335,1.3333333333333333,0,0.3333333333333333,2,8,0,0,2,7,1,4,0,1,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,3,0,1.5,0,3,1.5,0,2,0,0,2,1,0,2,41,13.666666666666666,3,25,1,0,1,1.4142135623730951,0.3333333333333333,0,0.3333333333333333,0,1,0,0,0,1,3,0,3,2,0,2,0,1,0,2,2,1,1,0,1,0,1,0,1,1,1,111.75476019047653,85.93446576814091,65.35366093010323 +greeter.py,greeter.py,unit,1,9,1,0.5,0,1,5,2.5,1,2,5,2.5,1,2,9,17,6,14,31,44.039100017307746,1.4206161295905724,15,121.11360846386408,10.5,0.09523809523809523,1271.692888870573,70.64960493725405,0.03912613122154238,9,7,4,0,2,2.25,1.75,1,0,0.5,2,9,0,0,2,7,1,4,0,2,2,0,0.5,0,2,0.5,0,1,0,0,3,0,1.5,0,3,1.5,0,2,0,0,2,1,0,2,41,10.25,0,25,1,0,1,1.4142135623730951,0.25,0,0.25,0,1,0,0,0,1,3,0,3,2,0,2,0,1,0,2,2,1,1,0,1,0,1,0,1,1,1,109.31197095345856,82.51208570936267,63.92512921254886 +greeter.py,Greeter,class,2,9,1,0.5,0,1,4,2,1,2,4,2,1,2,9,17,6,14,31,44.039100017307746,1.4206161295905724,15,121.11360846386408,10.5,0.09523809523809523,1271.692888870573,70.64960493725405,0.03912613122154238,8,7,4,0,1,2.6666666666666665,2.3333333333333335,1.3333333333333333,0,0.3333333333333333,2,8,0,0,2,7,1,4,0,1,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,3,0,1.5,0,3,1.5,0,2,0,0,2,1,0,2,41,13.666666666666666,3,25,1,0,1,1.4142135623730951,0.3333333333333333,0,0.3333333333333333,0,1,0,0,0,1,3,0,3,2,0,2,0,1,0,2,2,1,1,0,1,0,1,0,1,1,1,111.450056131092,85.49487073272815,65.17547142169123 greeter.py,__init__,function,3,4,0,0,0,0,1,1,1,1,1,1,1,1,6,6,3,6,12,20.264662506490403,1.6887218755408668,9,38.039100017307746,6,0.16666666666666666,228.23460010384647,12.679700005769249,0.012448959890820123,2,2,1,0,0,2,2,1,0,0,2,2,0,0,2,2,1,1,0,0,1,0,1,0,1,1,1,1,0,0,2,0,2,0,2,2,2,2,0,0,0,0,0,0,13,13,13,13,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,140.6202198664881,127.27306171748977,82.2340467055486 greeter.py,greet,function,6,9,1,1,1,1,2,2,2,2,2,2,2,2,6,9,4,7,16,23.509775004326936,1.4693609377704335,10,53.150849518197795,5.25,0.19047619047619047,279.0419599705384,15.502331109474357,0.014233938588537278,4,4,3,0,0,4,4,3,0,0,4,4,0,0,4,4,3,3,0,0,1,0,1,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,2,2,2,2,25,25,25,25,0,0,1,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,127.4217340918969,108.33349160264797,74.51563397186952 From 501591cf5be5b14a870fb3f6ac57ba19206a5575 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:26:34 -0700 Subject: [PATCH 27/48] fix(getter/python): unbill an f-string's directive = A self-documenting f-string field (`f"{z=}"`, Python 3.8+) billed its `=` as a Halstead operator, so `bca ops` reported `["="]` for a line that assigns nothing. The `=` is a directive of the replacement field: it asks for the expression's source text to be printed before its value. It belongs with the `{` opener and the format-spec `:` that #1486 left unbilled under the #1314 interpolation-opener rule, and with the `!r` conversion, which was never billed. So it is withdrawn under exactly the parents the `{` is: `Interpolation`, and `FormatExpression` for the nested `f"{a:{w=}}"` form Python 3.12 accepts. The helper is renamed from `brace_or_colon_role` to `replacement_field_role`, and `EQ` moves out of the main operator arm, where it would be dead. The guard reads the parent, so a real `=` inside a field is still billed: a keyword argument (`f"{g(a=1)}"`) keeps its `=`, and a walrus (`f"{(y := 1)}"`) keeps its `:=`, a different token. ABC's A axis never counted this `=`: Python ABC counts `Assignment` nodes that carry an `EQ` child, and the directive's parent is the interpolation. `f"{z=}"` measures A = 0 before and after this change, so only the getter changes. Fixes #1551 --- big-code-analysis-ast/src/getter/python.rs | 28 ++++++++++++---------- big-code-analysis-book/src/metrics.md | 4 +++- src/metrics/halstead.rs | 15 ++++++++++-- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/big-code-analysis-ast/src/getter/python.rs b/big-code-analysis-ast/src/getter/python.rs index 49f41b2b..1396941e 100644 --- a/big-code-analysis-ast/src/getter/python.rs +++ b/big-code-analysis-ast/src/getter/python.rs @@ -3,8 +3,8 @@ use super::*; -/// The role of a `{` or `:`, which Python spells both as syntax and -/// inside an f-string's replacement field. +/// The role of a `{`, `:` or `=`, which Python spells both as syntax +/// and inside an f-string's replacement field. /// /// As syntax each is an operator (#1486). The interpolation opener /// (`f"{x}"`, and a nested `{w}` inside a format spec) is spelling, not @@ -12,12 +12,15 @@ use super::*; /// and Python follows them. The `:` that opens a format spec /// (`f"{x:>10}"`) introduces the format mini-language, whose text is /// string content and counts as nothing, so its delimiter does not -/// count either. -fn brace_or_colon_role(kind: Python, parent: Option) -> TokenRole { +/// count either. The self-documenting `=` (`f"{x=}"`) is a directive of +/// the same field: it asks for the expression's text to be printed and +/// assigns nothing, like the `!r` conversion beside it, which was +/// already unbilled (#1551). +fn replacement_field_role(kind: Python, parent: Option) -> TokenRole { use Python::*; match (kind, parent) { - (LBRACE, Some(Interpolation | FormatExpression)) | (COLON, Some(FormatSpecifier)) => { + (LBRACE | EQ, Some(Interpolation | FormatExpression)) | (COLON, Some(FormatSpecifier)) => { TokenRole::Unknown } _ => TokenRole::Operator, @@ -38,13 +41,14 @@ impl Getter for PythonCode { use Python::*; let kind: Python = node.kind_id().into(); - // Bracket openers, `;` and `:` are vocabulary like every other - // getter's (#1395, #1486); only the f-string replacement-field - // spellings are withdrawn. An early return rather than an arm, - // because the match below is one rustfmt declines to format. - if matches!(kind, LBRACE | COLON) { + // Bracket openers, `;`, `:` and `=` are vocabulary like every + // other getter's (#1395, #1486); only the f-string + // replacement-field spellings are withdrawn. An early return + // rather than an arm, because the match below is one rustfmt + // declines to format. + if matches!(kind, LBRACE | COLON | EQ) { let parent = ancestors.parent(node).map(|p| p.kind_id().into()); - return brace_or_colon_role(kind, parent); + return replacement_field_role(kind, parent); } match kind { @@ -61,7 +65,7 @@ impl Getter for PythonCode { }, Import | DOT | From | COMMA | As | STAR | GTGT | Assert | COLONEQ | Return | Def | Del | Raise | Pass | Break | Continue | If | Elif | Else | Async | For - | While | Try | Except | Finally | With | DASHGT | EQ | Global | Nonlocal | Exec + | While | Try | Except | Finally | With | DASHGT | Global | Nonlocal | Exec | AT | And | Or | PLUS | DASH | SLASH | PERCENT | SLASHSLASH | STARSTAR | PIPE | AMP | CARET | LTLT | TILDE | LT | LTEQ | EQEQ | BANGEQ | GTEQ | GT | LTGT | PLUSEQ | DASHEQ | STAREQ | SLASHEQ | ATEQ | SLASHSLASHEQ | PERCENTEQ | STARSTAREQ diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index c2af115a..e785f15d 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -691,7 +691,9 @@ tokens involved the same way they spell real operators: `$"{x}"` in C# and `f"{x}"` in Python all count the interpolated expression's own operators and nothing for the opener itself. The `:` that opens a Python format spec (`f"{x:>10}"`) counts nothing - either, because the spec after it is string content (#1486). + either, because the spec after it is string content (#1486). Nor + does the self-documenting `=` of `f"{x=}"`, which prints the + expression's text and assigns nothing (#1551). - **A delimiter around *syntax* is an operator, whatever it delimits.** The rule above is about a literal's own punctuation, and it stops there. Halstead counts the vocabulary a program is written in and has diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index dba2ee86..f0985a1f 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -5959,8 +5959,10 @@ end", /// #1486: an f-string replacement field's `{` and format-spec `:` are /// spelling, not operations, like every other language's /// interpolation opener (#1314) — the f-string rows bill nothing. - /// The guard reads the token's *parent*, so a dict written inside a - /// replacement field keeps its own `{}` and `:`. + /// #1551 adds the self-documenting `=` (`f"{z=}"`), a directive of + /// the same field. The guard reads the token's *parent*, so a dict, + /// a keyword argument or a walrus written inside a replacement field + /// keeps its own operators. #[cfg(feature = "python")] #[test] fn python_f_string_delimiters_are_not_operators_1486() { @@ -5971,6 +5973,15 @@ end", // `{}`, `:`, `[]` of the dict and its subscript, nothing of // the replacement field around them ("dict inside a field", "f\"{ {k: v}[k] }\"\n", [3, 3]), + ("self-documenting =", "f\"{z=}\"\n", [0, 0]), + // the `=` keeps its spaces in the output; `!r` and `:` follow it + ("spaced = with conversion", "f\"{z = !r:>10}\"\n", [0, 0]), + // a `=` in a nested format expression (Python 3.12+) + ("nested self-documenting =", "f\"{a:{w=}}\"\n", [0, 0]), + // `()`, `=` — a keyword argument's `=` is a real operator + ("keyword argument inside a field", "f\"{g(a=1)}\"\n", [2, 2]), + // `()`, `:=` + ("walrus inside a field", "f\"{(y := 1)}\"\n", [2, 2]), ]; assert_operator_rows::("foo.py", ROWS); } From 3c96608d436a082b0497f2687f6081a24d534eb8 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:51:38 -0700 Subject: [PATCH 28/48] fix(getter): bill type-declaration keywords Java, the JS family, Groovy and C++ billed no Halstead operator for `class`, while C#, Kotlin, PHP, Ruby, Python and Rust (`struct`) bill their declaration keyword. JavaScript billed `function` but not `class`. Each getter now bills the keyword leaf and leaves the declaration wrapper unlisted, so a declaration bills its keyword once. In the JS family `Class2` is the leaf; the unsuffixed `Class` is the class expression node and stays unbilled. Keywords beyond `class`, each to close the same gap within one getter or between grammars that share an alphabet: - Java: `interface`, `enum`, `record`, `@interface`, as C# bills its `interface` / `enum` / `record`. - TypeScript / TSX: `interface`, `enum`, for the same reason. - Groovy: `interface`, `trait`, `enum`, `record`, mirroring Java. - C++ / Mozcpp: `struct`, `union`, `enum`. `struct S {}` and `class S {}` are the same declaration and must score alike. - C: `struct`, `union`, `enum`. c.rs defines C's set as C++'s minus the C++-only forms, so a header must not score by which grammar parsed it. - ObjC: C's three, plus `@interface`, `@implementation` and `@protocol`, its class and protocol declarations. `@class` already billed its `@` and is unchanged. Two positions where `class` declares no class stay unbilled: Java's `C.class` is already one operand (grammar-dispatch section 5), and C++'s `template ` is the synonym of `typename T`, which is unbilled. TypeScript `type` / `namespace` / `module`, C++ `namespace` / `typedef` and Kotlin `interface` are left alone. The corpus snapshots move in halstead.* and mi.* only: 449 C++ and 178 JavaScript files. No Java, Groovy, TypeScript, C or ObjC corpus is snapshotted. The self-scan baseline does not move. Fixes #1552 --- .rustfmt-bail-baseline.txt | 4 +- big-code-analysis-ast/src/getter.rs | 6 + big-code-analysis-ast/src/getter/c.rs | 7 +- big-code-analysis-ast/src/getter/cpp.rs | 28 ++ big-code-analysis-ast/src/getter/groovy.rs | 30 +- big-code-analysis-ast/src/getter/java.rs | 10 + big-code-analysis-ast/src/getter/mozcpp.rs | 18 + big-code-analysis-ast/src/getter/objc.rs | 15 +- big-code-analysis-ast/src/getter/tsx.rs | 6 +- .../src/getter/typescript.rs | 6 +- big-code-analysis-book/src/metrics.md | 8 +- src/metrics/halstead.rs | 410 +++++++++++++----- src/ops.rs | 5 +- ...put_formats__csv_test__csv_cpp_widget.snap | 6 +- tests/parity/cpp_mozcpp_parity.rs | 11 +- tests/repositories/big-code-analysis-output | 2 +- 16 files changed, 444 insertions(+), 128 deletions(-) diff --git a/.rustfmt-bail-baseline.txt b/.rustfmt-bail-baseline.txt index c9d031d0..b14584e0 100644 --- a/.rustfmt-bail-baseline.txt +++ b/.rustfmt-bail-baseline.txt @@ -129,9 +129,9 @@ big-code-analysis-ast/src/getter/cpp.rs 4 big-code-analysis-ast/src/getter/csharp.rs 9 big-code-analysis-ast/src/getter/elixir.rs 3 big-code-analysis-ast/src/getter/go.rs 1 -big-code-analysis-ast/src/getter/groovy.rs 7 +big-code-analysis-ast/src/getter/groovy.rs 9 big-code-analysis-ast/src/getter/irules.rs 6 -big-code-analysis-ast/src/getter/java.rs 5 +big-code-analysis-ast/src/getter/java.rs 6 big-code-analysis-ast/src/getter/kotlin.rs 4 big-code-analysis-ast/src/getter/lua.rs 3 big-code-analysis-ast/src/getter/mozcpp.rs 4 diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index b9f01a26..90d77eaa 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -247,6 +247,12 @@ macro_rules! impl_js_family_get_op_type { | CARET | CARETEQ | PIPEEQ | Yield | LBRACK | LBRACE | Await | QMARK | QMARKQMARK | EQGT | DOTDOTDOT | New | Let | Var | Const | Function | FunctionExpression | SEMI | Typeof | Instanceof | Void + // `Class2` is the `class` keyword leaf, billed beside + // `function` (#1552). The unsuffixed `Class` is the + // class *expression* node that wraps it + // (`const C = class {}`), so it stays unlisted and an + // expression bills one `class`, not two. + | Class2 // `get`/`set` accessor keywords are operators, matching the // C# getter's `Get | Set | Init | Add | Remove` accessor arm. // Before #695 the JS family classified them as operands, so diff --git a/big-code-analysis-ast/src/getter/c.rs b/big-code-analysis-ast/src/getter/c.rs index 5c3e0a77..9aa3bd22 100644 --- a/big-code-analysis-ast/src/getter/c.rs +++ b/big-code-analysis-ast/src/getter/c.rs @@ -80,7 +80,12 @@ impl Getter for CCode { // modifier has a distinct kind_id, so keying by kind_id (the default // `operators` store) keeps them distinct in n1 while `long long`'s // two `long` tokens correctly fold to one n1 entry but two N1 hits. - | Signed | Unsigned | Long | Short => TokenRole::Operator, + | Signed | Unsigned | Long | Short + // `struct` / `union` / `enum`, which C++ bills since #1552. + // They are C keywords, not C++-only forms, so leaving them + // out here would score a header differently by the grammar + // that parsed it. The derivation is on the Cpp arm. + | Struct | Union | Enum => TokenRole::Operator, // `CharLiteral` joins the operand list here for the whole // C family — `cpp.rs`, `mozcpp.rs` and `objc.rs` carry the // same arm and point back at this note (#1316). Before it a diff --git a/big-code-analysis-ast/src/getter/cpp.rs b/big-code-analysis-ast/src/getter/cpp.rs index be9368a7..4841d746 100644 --- a/big-code-analysis-ast/src/getter/cpp.rs +++ b/big-code-analysis-ast/src/getter/cpp.rs @@ -109,6 +109,25 @@ impl Getter for CppCode { { TokenRole::Unknown } + // `class` in a template parameter list (`template `, + // `class U = int`, `class... Ts`) declares a type parameter, + // not a class: it is the exact synonym of `typename`, which + // is unbilled, so it stays unbilled too and the two + // spellings score alike. Everywhere else — a class + // specifier, `enum class`, a forward declaration — it bills + // as the declaration keyword it is (#1552). + Class + if matches!( + ancestors.parent(node).map(|p| p.kind_id().into()), + Some( + TypeParameterDeclaration + | OptionalTypeParameterDeclaration + | VariadicTypeParameterDeclaration + ) + ) => + { + TokenRole::Unknown + } DOT | DOTSTAR | LPAREN | LPAREN2 | COMMA | STAR | GTGT | COLON | SEMI | Return | Break | Continue | If | Else | Switch | Case | Default | For | While | Goto | Do | Delete | New | Try | Try2 | Catch | Throw | EQ | AMPAMP | PIPEPIPE | DASH @@ -165,6 +184,15 @@ impl Getter for CppCode { // `co_return_statement` / `co_yield_statement`, which no arm // classifies, so nothing bills one twice. | CoReturn | CoYield + // The type-declaration keywords, billed as Rust bills + // `struct` and C# its `class` / `struct` / `enum` (#1552). + // Each is the keyword leaf of a `class_specifier` / + // `struct_specifier` / `union_specifier` / `enum_specifier`, + // none of which is classified, so a specifier bills its + // keyword once — and `enum class` bills both of its two. + // The same leaf heads an elaborated type (`struct S s;`), + // where it is billed too, as `int` is in `int s;`. + | Class | Struct | Union | Enum => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only diff --git a/big-code-analysis-ast/src/getter/groovy.rs b/big-code-analysis-ast/src/getter/groovy.rs index 02aa49a5..5f5d5be2 100644 --- a/big-code-analysis-ast/src/getter/groovy.rs +++ b/big-code-analysis-ast/src/getter/groovy.rs @@ -83,11 +83,12 @@ impl Getter for GroovyCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Groovy::*; // Mirrors `JavaCode`'s minimal classification — modifiers - // (`Public`, `Static`, …), declaration keywords (`Class`, - // `Interface`, …), and module keywords (`Package`, `Import`, - // …) are excluded because they live inside `Modifiers` / - // `*Declaration` wrappers and would over-count if treated as - // separate operators. The dekobon Groovy grammar (#246, #247) + // (`Public`, `Static`, …) and module keywords (`Package`, + // `Import`, …) are excluded. The type-declaration keywords + // (`class`, `interface`, `trait`, `enum`, `record`) are billed, + // as Java's are since #1552: no `*Declaration` wrapper is + // classified, so the keyword leaf is the only node that can + // bill one. The dekobon Groovy grammar (#246, #247) // emits a distinct named node for every Groovy-specific // operator (Elvis `?:`, safe-nav `?.`, identity `===`/`!==`, // regex `=~`/`==~`, spaceship `<=>`, exclusive ranges @@ -156,12 +157,31 @@ impl Getter for GroovyCode { Super if ancestors.parent_has_kind(node, Wildcard as u16) => TokenRole::Operator, Super => TokenRole::Operand, + // `trait` and `record` are contextual keywords, so valid + // Groovy may use either as a name. Inside a class the grammar + // parses that name as an `identifier`, but at script level it + // fails `record = 3`, `String record` and `void record() {}` + // into an `ERROR` holding the keyword leaf. Billing the leaf + // there would make a variable an operator, so it bills only + // under the declaration it introduces (#1552). + Trait if ancestors.parent_has_kind(node, TraitDeclaration as u16) => { + TokenRole::Operator + } + Record if ancestors.parent_has_kind(node, RecordDeclaration as u16) => { + TokenRole::Operator + } + // Control-flow + keyword operators (mirrors Java's set, // minus tokens that no longer exist in the dekobon grammar // — `This`, `VoidType`, `Throws2`). If | Else | Switch | Case | Try | Catch | Throw | Throws | For | While | Continue | Break | Do | Finally | New | Return | Default | Abstract | Assert | Instanceof | Extends | Final | Implements | Transient | Synchronized | Def | In | As + // Type-declaration keywords (#1552). Unlike Java's, `class` + // needs no class-literal guard: `String.class` parses as a + // `field_access` whose `class` is a plain `identifier`. + // `trait` and `record` are the gated arms above. + | Class | Interface | Enum // Separators / brackets. | SEMI | COMMA | COLONCOLON | DOT | DASHGT | LBRACE | LBRACK | LPAREN // Java-compatible operators (arithmetic, bitwise, comparison, assignment). diff --git a/big-code-analysis-ast/src/getter/java.rs b/big-code-analysis-ast/src/getter/java.rs index 20e35167..1cd8bb5b 100644 --- a/big-code-analysis-ast/src/getter/java.rs +++ b/big-code-analysis-ast/src/getter/java.rs @@ -84,12 +84,22 @@ impl Getter for JavaCode { // keywords, operators, literals: https://docs.oracle.com/javase/specs/jls/se18/html/jls-3.html#jls-3.12 // https://www.geeksforgeeks.org/software-engineering-halsteads-software-metrics/?msclkid=5e181114abef11ecbb03527e95a34828 match node.kind_id().into() { + // `class` declares a class (billed below, #1552) except as the + // tail of a class literal (`C.class`): that `class_literal` is + // already one operand keyed by its whole text, so billing its + // leaf too would pay the same bytes twice (grammar-dispatch + // section 5, the compound-leaf guard). + Class if ancestors.parent_has_kind(node, ClassLiteral as u16) => TokenRole::Unknown, // Operator: control flow | If | Else | Switch | Case | Try | Catch | Throw | Throws | Throws2 | For | While | Continue | Break | Do | Finally // Operator: keywords | New | Return | Default | Abstract | Assert | Instanceof | Extends | Final | Implements | Transient | Synchronized | VoidType + // Operator: type-declaration keywords, billed as C# bills + // its own (#1552). Only the leaves: no `*_declaration` + // wrapper is listed, so a declaration bills its keyword once. + | Class | Interface | Enum | Record | ATinterface // Operator: brackets and comma and terminators (separators) | SEMI | COMMA | COLONCOLON | DOT | DASHGT | LBRACE | LBRACK | LPAREN // Operator: operators diff --git a/big-code-analysis-ast/src/getter/mozcpp.rs b/big-code-analysis-ast/src/getter/mozcpp.rs index 6d37967f..a7a0e88c 100644 --- a/big-code-analysis-ast/src/getter/mozcpp.rs +++ b/big-code-analysis-ast/src/getter/mozcpp.rs @@ -88,6 +88,21 @@ impl Getter for MozcppCode { { TokenRole::Unknown } + // `class` in a template parameter list is `typename`'s + // synonym and stays unbilled — the derivation is on the Cpp + // twin (#1552). + Class + if matches!( + ancestors.parent(node).map(|p| p.kind_id().into()), + Some( + TypeParameterDeclaration + | OptionalTypeParameterDeclaration + | VariadicTypeParameterDeclaration + ) + ) => + { + TokenRole::Unknown + } DOT | DOTSTAR | LPAREN | LPAREN2 | COMMA | STAR | GTGT | COLON | SEMI | Return | Break | Continue | If | Else | Switch | Case | Default | For | While | Goto | Do | Delete | New | Try | Try2 | Catch | Throw | EQ | AMPAMP | PIPEPIPE | DASH @@ -116,6 +131,9 @@ impl Getter for MozcppCode { // `co_return` / `co_yield`, billed as `return` is — the // derivation is on the Cpp twin (#1547). | CoReturn | CoYield + // The type-declaration keywords — the derivation is on the + // Cpp twin (#1552). + | Class | Struct | Union | Enum => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only diff --git a/big-code-analysis-ast/src/getter/objc.rs b/big-code-analysis-ast/src/getter/objc.rs index 902ea448..ead1313c 100644 --- a/big-code-analysis-ast/src/getter/objc.rs +++ b/big-code-analysis-ast/src/getter/objc.rs @@ -93,6 +93,16 @@ impl Getter for ObjcCode { AT if ancestors.parent_has_kind(node, StringLiteral as u16) => TokenRole::Unknown, // The C operator set, then the ObjC-specific structural // keywords / markers from `In` onwards. + // + // From `Struct` onwards come the type-declaration keywords + // (#1552): C's `struct` / `union` / `enum` (see `c.rs`), and + // ObjC's own class and protocol declarations. Each is the + // leaf of a `class_interface` / `class_implementation` / + // `protocol_declaration`, none of which is classified, so a + // declaration bills its keyword once. `@class Fwd;` needs no + // arm: it parses as an `@` token, already billed here, and a + // separate `class` leaf, so the directive already bills one + // operator. DOT | LPAREN | LPAREN2 | COMMA | STAR | GTGT | COLON | SEMI | Return | Break | Continue | If | Else | Switch | Case | Default | For | While | Goto | Do | EQ | AMPAMP | PIPEPIPE | DASH | DASHDASH | DASHGT | PLUS | PLUSPLUS | SLASH | PERCENT @@ -100,9 +110,8 @@ impl Getter for ObjcCode { | DASHEQ | BANG | STAREQ | SLASHEQ | PERCENTEQ | GTGTEQ | LTLTEQ | AMPEQ | CARET | CARETEQ | PIPEEQ | LBRACK | LBRACE | QMARK | PrimitiveType | TypeSpecifier | Sizeof | Signed | Unsigned | Long | Short | In | AT | ATtry | ATcatch | ATfinally - | ATthrow | ATsynchronized | ATautoreleasepool | ATselector | ATencode => { - TokenRole::Operator - } + | ATthrow | ATsynchronized | ATautoreleasepool | ATselector | ATencode | Struct + | Union | Enum | ATinterface | ATimplementation | ATprotocol => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only // classified node in a character literal, so it bills one diff --git a/big-code-analysis-ast/src/getter/tsx.rs b/big-code-analysis-ast/src/getter/tsx.rs index ad01d6ba..13a2b6ad 100644 --- a/big-code-analysis-ast/src/getter/tsx.rs +++ b/big-code-analysis-ast/src/getter/tsx.rs @@ -74,9 +74,13 @@ impl Getter for TsxCode { // delimiters. TSX is the only one of the two TypeScript grammars // with JSX, so the `.ts` invocation has no such variants to pass // (#1395), and the same holds for the `jsx:` text kinds (#1483). + // + // `Interface` / `Enum` are the TS-only type-declaration keyword + // leaves, billed as the shared `class` is (#1552) and as C# and + // Java bill theirs; their `*_declaration` wrappers stay unlisted. impl_js_family_get_op_type!( Tsx, - op_extras: [QMARKDOT, PredefinedType, LTSLASH, SLASHGT], + op_extras: [QMARKDOT, PredefinedType, Interface, Enum, LTSLASH, SLASHGT], operand_extras: [Identifier2, String2], predefined_void: PredefinedType, jsx: [JsxText, HtmlCharacterReference], diff --git a/big-code-analysis-ast/src/getter/typescript.rs b/big-code-analysis-ast/src/getter/typescript.rs index 3027db51..210b4e6b 100644 --- a/big-code-analysis-ast/src/getter/typescript.rs +++ b/big-code-analysis-ast/src/getter/typescript.rs @@ -75,9 +75,13 @@ impl Getter for TypescriptCode { // // No `jsx:` argument: the `.ts` grammar's enum carries `JsxText` in // its shared externals, but no production emits it (#1483). + // + // `Interface` / `Enum` are the TS-only type-declaration keyword + // leaves, billed as the shared `class` is (#1552) and as C# and + // Java bill theirs; their `*_declaration` wrappers stay unlisted. impl_js_family_get_op_type!( Typescript, - op_extras: [QMARKDOT, PredefinedType], + op_extras: [QMARKDOT, PredefinedType, Interface, Enum], operand_extras: [], predefined_void: PredefinedType, ); diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index e785f15d..e5f73b85 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -711,7 +711,13 @@ tokens involved the same way they spell real operators: and label `:`, which is a separate operator from `::`. Both languages left these out until #1486, so their Halstead scores rose with it. Python's `class` keyword followed in #1550: it is now billed like - `def`, as Ruby, Kotlin, C# and PHP bill their `class`. + `def`, as Ruby, Kotlin, C# and PHP bill their `class`. Java, Groovy, + the JavaScript family, C, C++ and Objective-C followed in #1552 with + their type-declaration keywords: `class`, `struct`, `union`, `enum`, + `interface`, `record`, `trait`, `@interface`, `@implementation` and + `@protocol`. A `class` that declares no class is not billed: Java's + `C.class` stays one operand, and C++'s `template ` scores + like `template `. - **An alternative spelling is an operator of its own.** C++ spells `&&` equally as `and`, `!=` as `not_eq` and so on through all eleven [alternative tokens](https://en.cppreference.com/w/cpp/language/operator_alternative). diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index f0985a1f..e27154b1 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -1106,13 +1106,13 @@ mod tests { #[test] fn cpp_dot_star_is_halstead_operator() { check_metrics::("struct S { void operator.*(int); };", "foo.cpp", |metric| { - // Unique operators with fix: {}, ;, (), int, void, .*, and - // the `operator` keyword (#1296). + // Unique operators with fix: {}, ;, (), int, void, .*, the + // `operator` keyword (#1296) and `struct` (#1552). // `.*` is the regression target — without the fix it - // falls through to `Unknown` and `u_operators` is 6. + // falls through to `Unknown` and `u_operators` is 7. // Unique operands: S let s = &metric.halstead; - assert_eq!(s.unique_operators(), 7); + assert_eq!(s.unique_operators(), 8); assert_eq!(s.unique_operands(), 1); }); } @@ -1135,12 +1135,12 @@ mod tests { "foo.cpp", |metric| { // Unique operators with fix: {}, ;, (), int, void, ->*, - // and the `operator` keyword (#1296). + // the `operator` keyword (#1296) and `struct` (#1552). // `->*` is the regression target — without the fix it - // falls through to `Unknown` and `u_operators` is 6. + // falls through to `Unknown` and `u_operators` is 7. // Unique operands: S let s = &metric.halstead; - assert_eq!(s.unique_operators(), 7); + assert_eq!(s.unique_operators(), 8); assert_eq!(s.unique_operands(), 1); }, ); @@ -2296,10 +2296,9 @@ mod tests { // // expected, for `class C { #x = 1; m() { return this.#x; } }`: // - // * Operators: `{`×2 (class body, method body), `=`, `;`×2, `(`, - // `return`, `.` — 8 total, 6 unique. (`class` is not in the - // JS-family operator arm, so it contributes nothing; that is - // pre-existing and unrelated.) + // * Operators: `class`, `{`×2 (class body, method body), `=`, + // `;`×2, `(`, `return`, `.` — 9 total, 7 unique. (`class` joined + // the JS-family operator arm in #1552.) // * Operands: `C`, `#x`, `1`, `m`, `this`, `#x` — 6 total, 5 // unique under JS/MozJS. Under TS/TSX the class *name* `C` // parses as `type_identifier`, which those getters do not @@ -2311,14 +2310,14 @@ mod tests { #[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))] const SRC: &str = "class C { #x = 1; m() { return this.#x; } }"; let check_js = |m: crate::CodeMetrics| { - assert_eq!(m.halstead.unique_operators(), 6); - assert_eq!(m.halstead.total_operators(), 8); + assert_eq!(m.halstead.unique_operators(), 7); + assert_eq!(m.halstead.total_operators(), 9); assert_eq!(m.halstead.unique_operands(), 5); assert_eq!(m.halstead.total_operands(), 6); }; let check_ts = |m: crate::CodeMetrics| { - assert_eq!(m.halstead.unique_operators(), 6); - assert_eq!(m.halstead.total_operators(), 8); + assert_eq!(m.halstead.unique_operators(), 7); + assert_eq!(m.halstead.total_operators(), 9); assert_eq!(m.halstead.unique_operands(), 4); assert_eq!(m.halstead.total_operands(), 5); }; @@ -2679,26 +2678,26 @@ mod tests { }", "foo.java", |metric| { - // Operators (n1=11): {} void () [] , . ; int = + / + // Operators (n1=12): class {} void () [] , . ; int = + / // Operands (n2=12): Main main args a b c avg 5 3 MessageFormat format "{0}" insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 11, - "total_operators": 26, + "unique_operators": 12, + "total_operators": 27, "unique_operands": 12, "total_operands": 22, - "length": 48, - "estimated_program_length": 81.07329781366414, - "purity_ratio": 1.6890270377846697, - "vocabulary": 23, - "volume": 217.13097389073664, - "difficulty": 10.083333333333334, - "level": 0.09917355371900825, - "effort": 2189.4039867315946, - "time": 121.63355481842193, - "bugs": 0.05620341201461669 + "length": 49, + "estimated_program_length": 86.03910001730775, + "purity_ratio": 1.7559000003532192, + "vocabulary": 24, + "volume": 224.66316253533665, + "difficulty": 11.0, + "level": 0.09090909090909091, + "effort": 2471.294787888703, + "time": 137.29415488270573, + "bugs": 0.060929616893854746 } "# ); @@ -2725,25 +2724,26 @@ mod tests { |metric| { // Verifies all 8 Java primitive-type keywords (byte, short, int, long, // char, float, double, boolean) are counted as distinct operators, and - // that true/false are counted as operands. + // that true/false are counted as operands. The other four operators + // are `class` (#1552), `{}`, `=` and `;`. insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 11, - "total_operators": 28, + "unique_operators": 12, + "total_operators": 29, "unique_operands": 19, "total_operands": 19, - "length": 47, - "estimated_program_length": 118.76437056043838, - "purity_ratio": 2.526901501285923, - "vocabulary": 30, - "volume": 230.62385799360038, - "difficulty": 5.5, - "level": 0.18181818181818182, - "effort": 1268.4312189648022, - "time": 70.46840105360012, - "bugs": 0.03905920146699976 + "length": 48, + "estimated_program_length": 123.730172764082, + "purity_ratio": 2.5777119325850415, + "vocabulary": 31, + "volume": 237.80142289857, + "difficulty": 6.0, + "level": 0.16666666666666666, + "effort": 1426.80853739142, + "time": 79.26714096619, + "bugs": 0.04224635182363915 } "# ); @@ -2762,14 +2762,15 @@ mod tests { #[test] fn java_self_and_super_references_are_operands() { let source = "class T {\n int f() { return this.x + super.y; }\n}"; - // Operators (n1=7, N1=9): {} x2, int, (), return, . x2, +, ; + // Operators (n1=8, N1=10): class (#1552), {} x2, int, (), + // return, . x2, +, ; // Operands (n2=6, N2=6): T, f, this, x, super, y - // Before the fix this read (9, 11, 4, 4) — `this` and `super` + // Before the fix this read (10, 12, 4, 4) — `this` and `super` // billed into the operator side of both counts. assert_halstead_counts::( source, "foo.java", - [7, 9, 6, 6], + [8, 10, 6, 6], "java self/super references", ); assert_keywords_are_operands_only::(source, "foo.java", &["this", "super"]); @@ -2944,26 +2945,32 @@ mod tests { // not already counted as operands, since `String` // was already an identifier in the prior grammar's // counting). - assert_eq!(metric.halstead.unique_operators(), 8); + // `class` is the ninth unique operator since #1552. + assert_eq!(metric.halstead.unique_operators(), 9); assert_eq!(metric.halstead.unique_operands(), 13); + // `bugs` is `effort.powf(2/3) / 3000`, and the platform libm's + // `powf` differs in the last ulp: macOS rendered this value + // as `…92342184` where Linux and Windows gave `…923421845`. + // Round it so the snapshot pins the metric, not the libm. insta::assert_json_snapshot!( metric.halstead, + { ".bugs" => insta::rounded_redaction(12) }, @r#" { - "unique_operators": 8, - "total_operators": 22, + "unique_operators": 9, + "total_operators": 23, "unique_operands": 13, "total_operands": 23, - "length": 45, - "estimated_program_length": 72.10571633583419, - "purity_ratio": 1.6023492519074265, - "vocabulary": 21, - "volume": 197.65428402504423, - "difficulty": 7.076923076923077, - "level": 0.14130434782608697, - "effort": 1398.7841638695438, - "time": 77.71023132608576, - "bugs": 0.04169134280255714 + "length": 46, + "estimated_program_length": 76.63504134881501, + "purity_ratio": 1.665979159756848, + "vocabulary": 22, + "volume": 205.13385445731566, + "difficulty": 7.961538461538462, + "level": 0.12560386473429952, + "effort": 1633.1810720255517, + "time": 90.73228177919731, + "bugs": 0.046227630923 } "# ); @@ -2997,29 +3004,29 @@ mod tests { // split), and `'x'` parses as `StringLiteral` (Groovy // single-quoted strings) rather than as // `CharacterLiteral`. Operators remaining in this - // fixture: `=` and `class`-body braces (only `{` is in - // the operator set). True/false collapse under one - // `BooleanLiteral`. - assert_eq!(metric.halstead.unique_operators(), 2); + // fixture: `=`, the `class` keyword (#1552) and the + // class-body braces (only `{` is in the operator set). + // True/false collapse under one `BooleanLiteral`. + assert_eq!(metric.halstead.unique_operators(), 3); assert_eq!(metric.halstead.unique_operands(), 27); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 2, - "total_operators": 10, + "unique_operators": 3, + "total_operators": 11, "unique_operands": 27, "total_operands": 28, - "length": 38, - "estimated_program_length": 130.38196255841365, - "purity_ratio": 3.4311042778529908, - "vocabulary": 29, - "volume": 184.60327781484773, - "difficulty": 1.037037037037037, - "level": 0.9642857142857143, - "effort": 191.44043625243467, - "time": 10.635579791801925, - "bugs": 0.01107221547116606 + "length": 39, + "estimated_program_length": 133.1368500605771, + "purity_ratio": 3.413765386168644, + "vocabulary": 30, + "volume": 191.36873322873222, + "difficulty": 1.5555555555555556, + "level": 0.6428571428571428, + "effort": 297.6846961335835, + "time": 16.53803867408797, + "bugs": 0.014861058740943886 } "# ); @@ -5873,7 +5880,19 @@ end", /// Asserts `[n1, N1]` for each `(label, source, expected)` row, one /// parse per row so a row's counts are its own. - #[cfg(any(feature = "python", feature = "rust"))] + #[cfg(any( + feature = "c", + feature = "cpp", + feature = "groovy", + feature = "java", + feature = "javascript", + feature = "mozcpp", + feature = "mozjs", + feature = "objc", + feature = "python", + feature = "rust", + feature = "typescript" + ))] fn assert_operator_rows(file: &str, rows: &[(&str, &str, [u64; 2])]) { for (label, source, expected) in rows { crate::test_support::check_func_space_only::( @@ -5986,6 +6005,188 @@ end", assert_operator_rows::("foo.py", ROWS); } + /// #1552: a JS-family `class` keyword bills as `function` does, once + /// per keyword. Shared by the four grammars of the macro; each runs + /// its own call so a grammar whose `Class2` stops being the keyword + /// leaf fails alone. Before #1552 every row billed one fewer. + #[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))] + const JS_FAMILY_CLASS_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}` + ("declaration", "class C {}\n", [2, 2]), + // `const`, `=`, `class`, `{}`, `;` — the class *expression* + // node wraps the keyword and bills nothing of its own + ("expression", "const D = class {};\n", [5, 5]), + // `class` twice, `extends`, `{}` twice + ("two classes", "class A {}\nclass B extends A {}\n", [3, 5]), + ]; + + #[cfg(any(feature = "javascript", feature = "mozjs"))] + #[test] + fn js_class_keyword_is_an_operator_1552() { + #[cfg(feature = "javascript")] + assert_operator_rows::("foo.js", JS_FAMILY_CLASS_ROWS); + #[cfg(feature = "mozjs")] + assert_operator_rows::("foo.js", JS_FAMILY_CLASS_ROWS); + } + + /// #1552 for the two TypeScript grammars: the shared `class` rows, + /// plus their own `interface` and `enum` declaration keywords. + #[cfg(feature = "typescript")] + #[test] + fn typescript_declaration_keywords_are_operators_1552() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `interface`, `{}` + ("interface", "interface I {}\n", [2, 2]), + // `enum`, `{}` + ("enum", "enum E { A }\n", [2, 2]), + ]; + assert_operator_rows::("foo.ts", JS_FAMILY_CLASS_ROWS); + assert_operator_rows::("foo.ts", ROWS); + assert_operator_rows::("foo.tsx", JS_FAMILY_CLASS_ROWS); + assert_operator_rows::("foo.tsx", ROWS); + } + + /// #1552: Java's type-declaration keywords are operators, as C#'s + /// are. The class-literal row pins the guard: `C.class` is one + /// operand, and its `class` leaf bills nothing of its own. + #[cfg(feature = "java")] + #[test] + fn java_declaration_keywords_are_operators_1552() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}` + ("class", "class C {}\n", [2, 2]), + // `interface`, `{}` + ("interface", "interface I {}\n", [2, 2]), + // `enum`, `{}` + ("enum", "enum E { A }\n", [2, 2]), + // `record`, `()`, `int`, `{}` + ("record", "record R(int x) {}\n", [4, 4]), + // `@interface`, `{}` + ("annotation type", "@interface N {}\n", [2, 2]), + // `class`, `{}`, `=`, `.`, `;` — one `class`: the literal's + // is part of the `C.class` operand + ("class literal", "class U { Object o = C.class; }\n", [5, 5]), + ]; + assert_operator_rows::("Foo.java", ROWS); + } + + /// #1552: Groovy's type-declaration keywords are operators, as + /// Java's are. `String.class` is a `field_access` whose `class` is a + /// plain identifier, so it stays an operand with no guard needed. + #[cfg(feature = "groovy")] + #[test] + fn groovy_declaration_keywords_are_operators_1552() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}` + ("class", "class C {}\n", [2, 2]), + // `interface`, `{}` + ("interface", "interface I {}\n", [2, 2]), + // `trait`, `{}` + ("trait", "trait T {}\n", [2, 2]), + // `enum`, `{}` + ("enum", "enum E { A }\n", [2, 2]), + // `record`, `()`, `{}` + ("record", "record R(int x) {}\n", [3, 3]), + // `def`, `=`, `.` — no `class` + ("class reference", "def a = String.class\n", [3, 3]), + // `=` twice: a script-level `record` / `trait` used as a name + // falls into error recovery with the keyword leaf, which is + // not a declaration and bills nothing + ("contextual names", "record = 3\ntrait = 4\n", [1, 2]), + ]; + assert_operator_rows::("foo.groovy", ROWS); + } + + /// #1552 rows shared by the two C++ clones. The template-parameter + /// rows pin the guard: `class T` is `typename T`'s synonym, and the + /// two spellings score alike. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const CPP_DECLARATION_KEYWORD_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}`, `;` + ("class", "class C {};\n", [3, 3]), + // `struct`, `{}`, `;` + ("struct", "struct S {};\n", [3, 3]), + // `union`, `{}`, `int`, `;` twice + ("union", "union U { int a; };\n", [4, 5]), + // `enum`, `{}`, `;` + ("enum", "enum E { A };\n", [3, 3]), + // `enum`, `class`, `{}`, `;` + ("enum class", "enum class F { B };\n", [4, 4]), + // `struct`, `;` — an elaborated type bills its keyword + ("elaborated type", "struct S s;\n", [2, 2]), + // `<`, `>`, `struct`, `{}`, `;` — no `class` + ( + "class parameter", + "template struct W {};\n", + [5, 5], + ), + ( + "typename parameter", + "template struct W {};\n", + [5, 5], + ), + // `<`, `=`, `int`, `>`, `struct`, `{}`, `;` + ( + "optional class parameter", + "template struct W {};\n", + [7, 7], + ), + // `<`, `>`, `struct`, `{}`, `;` — `...` is an operand + ( + "variadic class parameter", + "template struct W {};\n", + [5, 5], + ), + ]; + + /// One call per clone, so reverting either arm fails its own row; + /// Mozcpp owns no file extension, so this is its only coverage. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_declaration_keywords_are_operators_1552() { + assert_operator_rows::("foo.cpp", CPP_DECLARATION_KEYWORD_ROWS); + assert_operator_rows::("foo.cpp", CPP_DECLARATION_KEYWORD_ROWS); + } + + /// #1552: C bills `struct` / `union` / `enum` as C++ does, so a + /// header scores the same whichever grammar parses it. + #[cfg(feature = "c")] + #[test] + fn c_declaration_keywords_are_operators_1552() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `struct`, `{}`, `int`, `;` twice + ("struct", "struct S { int a; };\n", [4, 5]), + // `union`, `{}`, `int`, `;` twice + ("union", "union U { int a; };\n", [4, 5]), + // `enum`, `{}`, `;` + ("enum", "enum E { A };\n", [3, 3]), + // `struct`, `;` + ("elaborated type", "struct S s;\n", [2, 2]), + ]; + assert_operator_rows::("foo.c", ROWS); + } + + /// #1552: ObjC bills C's `struct` and its own class and protocol + /// declaration keywords. `@class` already billed its `@` and stays + /// as it was. + #[cfg(feature = "objc")] + #[test] + fn objc_declaration_keywords_are_operators_1552() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `@interface`, `:` + ("interface", "@interface C : NSObject\n@end\n", [2, 2]), + // `@implementation` + ("implementation", "@implementation C\n@end\n", [1, 1]), + // `@protocol` + ("protocol", "@protocol P\n@end\n", [1, 1]), + // `struct`, `{}`, `int`, `;` twice + ("struct", "struct S { int a; };\n", [4, 5]), + // `@`, `;` — unchanged by #1552 + ("forward class", "@class Fwd;\n", [2, 2]), + ]; + assert_operator_rows::("foo.m", ROWS); + } + #[cfg(feature = "bash")] #[test] fn bash_operators_and_operands() { @@ -9887,13 +10088,14 @@ f() { #[cfg(feature = "objc")] #[test] fn objc_nsstring_literal_is_one_operand() { - // expected: [n1, N1, n2, N2]. Before the guard the first two rows - // read [8, 8, 5, 5] and [8, 9, 6, 6] — one `@` operator per - // literal; the boxing control is unchanged. + // expected: [n1, N1, n2, N2], each operator count including the + // wrapper's `@implementation` (#1552). Before the guard the first + // two rows read [9, 9, 5, 5] and [9, 10, 6, 6] — one `@` operator + // per literal; the boxing control is unchanged. let cases = [ - ("NSString *s = @\"str\";", [7, 7, 5, 5]), - ("NSString *t = @\"x\" @\"y\";", [7, 7, 6, 6]), - ("NSNumber *n = @42;", [8, 8, 5, 5]), + ("NSString *s = @\"str\";", [8, 8, 5, 5]), + ("NSString *t = @\"x\" @\"y\";", [8, 8, 6, 6]), + ("NSNumber *n = @42;", [9, 9, 5, 5]), ]; for (body, counts) in cases { let source = format!("@implementation Foo\n- (void)m {{\n {body}\n}}\n@end\n"); @@ -9927,32 +10129,33 @@ f() { @end "; check_metrics::(source, "foo.m", |metric| { - // n1 = 14 unique operators: + // n1 = 15 unique operators: // `&&`, `()`, `+`, `-`, `:`, `;`, `<`, `=`, `>`, - // `[]` (message send), `if`, `int`, `return`, `{}`. + // `[]` (message send), `if`, `int`, `return`, `{}`, + // `@implementation` (#1552). // The `@` of `@"hi"` is part of the literal's operand // key, not an operator (see `objc_nsstring_literal_is_one_operand`). // n2 = 10 unique operands: // `Foo`, `bar`, `log`, `self`, `x`, `y`, `0`, `1`, `10`, // `@"hi"` (the ObjC string literal). - assert_eq!(metric.halstead.unique_operators(), 14); + assert_eq!(metric.halstead.unique_operators(), 15); assert_eq!(metric.halstead.unique_operands(), 10); insta::assert_json_snapshot!(metric.halstead, @r#" { - "unique_operators": 14, - "total_operators": 22, + "unique_operators": 15, + "total_operators": 23, "unique_operands": 10, "total_operands": 14, - "length": 36, - "estimated_program_length": 86.52224985768008, - "purity_ratio": 2.403395829380002, - "vocabulary": 24, - "volume": 165.0586500259616, - "difficulty": 9.8, - "level": 0.1020408163265306, - "effort": 1617.5747702544238, - "time": 89.86526501413465, - "bugs": 0.04593266617952463 + "length": 37, + "estimated_program_length": 91.82263988300141, + "purity_ratio": 2.481692969810849, + "vocabulary": 25, + "volume": 171.8226790216648, + "difficulty": 10.5, + "level": 0.09523809523809523, + "effort": 1804.1381297274804, + "time": 100.22989609597113, + "bugs": 0.049399808887691035 } "#); }); @@ -10226,15 +10429,15 @@ f() { // Operators, keyed by kind_id except the text-keyed primitives: // `{` x3 (class body, `m1`, `m2`), `int` x3, `;` x4 (the field, // the two returns, the struct terminator), `(` x2, `return` x2, - // `->` x2, and the one `*` of `S* p`. `struct` is in no arm, and - // #695 dropped every closing delimiter. n1 = 7, N1 = 17. + // `->` x2, the one `*` of `S* p`, and `struct` (#1552). #695 + // dropped every closing delimiter. n1 = 8, N1 = 18. // // Operands, keyed by source text: `S` x2 (the name and the // parameter type), `x` x3, `m1`, `m2`, `p` x2, and — the fix — // `this` x1. n2 = 6, N2 = 10. Before #1361: n2 5, N2 9, with // every operator count identical, which is the whole claim. let counts = format!("{label}: this->x and p->x are the same shape"); - assert_halstead_counts::(CPP_THIS_RECEIVER_PARITY, file, [7, 17, 6, 10], &counts); + assert_halstead_counts::(CPP_THIS_RECEIVER_PARITY, file, [8, 18, 6, 10], &counts); assert_ops_operands::( CPP_THIS_RECEIVER_PARITY, file, @@ -10259,20 +10462,21 @@ S s; int a = s[1]; auto t = co_await s; #[cfg(any(feature = "cpp", feature = "mozcpp"))] #[track_caller] fn assert_cpp_operator_names(label: &str) { - // Operators (n1 = 16, N1 = 39): `operator` 5 (four + // Operators (n1 = 17, N1 = 40): `operator` 5 (four // `operator_name`s and the `operator_cast`); the names `[]` 1, // `()` 1, `""` 1; `co_await` 2 (the name and the expression); // the `(` of a parameter list 5; `;` 9; `int` 3; `long` 3; // `bool` 2; `=` 2; `{` 1; `void` 1; `unsigned` 1; `::` 1; the - // subscript `[` 1. `struct`, `auto` and `const` are in no arm. + // subscript `[` 1; `struct` 1 (#1552). `auto` and `const` are in + // no arm. // // Operands (n2 = 6, N2 = 9): `s` 3, `S` 2 (the qualifier in // `S::operator bool` is a namespace identifier, which #1096 // leaves unbilled), `_x`, `a`, `1`, `t`. // - // Before #1296: (11, 29, 6, 9), the five operator kinds billed - // nowhere. - assert_halstead_counts::(CPP_OPERATOR_NAMES, "ops.cpp", [16, 39, 6, 9], label); + // Before #1296: (12, 30, 6, 9) with `struct`, the five operator + // kinds billed nowhere. + assert_halstead_counts::(CPP_OPERATOR_NAMES, "ops.cpp", [17, 40, 6, 9], label); // `operator[]` / `operator()` are kinds of their own, so they sit // beside the subscript `[` and the parameter-list `(` as a second diff --git a/src/ops.rs b/src/ops.rs index 599ba938..aafe2ef5 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -889,7 +889,7 @@ mod tests { }", "foo.java", &mut [ - "{}", "void", "()", "[]", ",", ".", ";", "int", "=", "+", "/", + "class", "{}", "void", "()", "[]", ",", ".", ";", "int", "=", "+", "/", ], &mut [ "Main", @@ -926,8 +926,9 @@ mod tests { }", "foo.java", // All 8 primitive-type keywords must appear as distinct operators. - // true/false appear as operands. + // true/false appear as operands; `class` is billed since #1552. &mut [ + "class", "{}", ";", "=", diff --git a/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap b/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap index ab931ab1..833713df 100644 --- a/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap +++ b/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap @@ -3,8 +3,8 @@ source: tests/output_formats/csv_test.rs expression: out --- path,space_name,space_kind,start_line,end_line,cognitive.sum,cognitive.average,cognitive.min,cognitive.max,cyclomatic.sum,cyclomatic.average,cyclomatic.min,cyclomatic.max,cyclomatic.modified.sum,cyclomatic.modified.average,cyclomatic.modified.min,cyclomatic.modified.max,halstead.unique_operators,halstead.total_operators,halstead.unique_operands,halstead.total_operands,halstead.length,halstead.estimated_program_length,halstead.purity_ratio,halstead.vocabulary,halstead.volume,halstead.difficulty,halstead.level,halstead.effort,halstead.time,halstead.bugs,loc.sloc,loc.ploc,loc.lloc,loc.cloc,loc.blank,loc.sloc_average,loc.ploc_average,loc.lloc_average,loc.cloc_average,loc.blank_average,loc.sloc_min,loc.sloc_max,loc.cloc_min,loc.cloc_max,loc.ploc_min,loc.ploc_max,loc.lloc_min,loc.lloc_max,loc.blank_min,loc.blank_max,nom.functions,nom.closures,nom.functions_average,nom.closures_average,nom.total,nom.average,nom.functions_min,nom.functions_max,nom.closures_min,nom.closures_max,nargs.function_args,nargs.closure_args,nargs.function_args_average,nargs.closure_args_average,nargs.total,nargs.average,nargs.function_args_min,nargs.function_args_max,nargs.closure_args_min,nargs.closure_args_max,nexits.sum,nexits.average,nexits.min,nexits.max,tokens.sum,tokens.average,tokens.min,tokens.max,abc.assignments,abc.branches,abc.conditions,abc.magnitude,abc.assignments_average,abc.branches_average,abc.conditions_average,abc.assignments_min,abc.assignments_max,abc.branches_min,abc.branches_max,abc.conditions_min,abc.conditions_max,wmc.class_wmc_sum,wmc.interface_wmc_sum,wmc.total,npm.class_npm_sum,npm.interface_npm_sum,npm.class_methods,npm.interface_methods,npm.class_coa,npm.interface_coa,npm.total,npm.total_methods,npm.coa,npa.class_npa_sum,npa.interface_npa_sum,npa.class_attributes,npa.interface_attributes,npa.class_cda,npa.interface_cda,npa.total,npa.total_attributes,npa.cda,mi.original,mi.sei,mi.visual_studio -widget.cc,widget.cc,unit,1,10,0,0,0,0,5,2.5,1,1,5,2.5,1,1,8,18,6,9,27,39.50977500432694,1.4633250001602571,14,102.79858289555531,6,0.16666666666666666,616.7914973733318,34.266194298518435,0.024152995332196794,10,9,2,0,1,2,1.8,0.4,0,0.2,1,10,0,0,1,9,1,2,0,1,2,0,0.4,0,2,0.4,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,7.6,0,12,1,0,0,1,0.2,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,108.457709340607,81.2796467168799,63.42556101789883 -widget.cc,ns,namespace,2,10,0,0,0,0,4,2,1,1,4,2,1,1,8,18,6,9,27,39.50977500432694,1.4633250001602571,14,102.79858289555531,6,0.16666666666666666,616.7914973733318,34.266194298518435,0.024152995332196794,9,9,2,0,0,2.25,2.25,0.5,0,0,1,9,0,0,1,9,1,2,0,0,2,0,0.5,0,2,0.5,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,9.5,5,12,1,0,0,1,0.25,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,110.3945496942638,83.97209683068972,64.55821619547591 -widget.cc,Widget,class,3,9,0,0,0,0,3,1.5,1,1,3,1.5,1,1,8,16,5,8,24,35.60964047443681,1.4837350197682004,13,88.81055323538621,6.4,0.15625,568.3875407064718,31.57708559480399,0.022872226594196698,7,7,2,0,0,2.3333333333333335,2.3333333333333335,0.6666666666666666,0,0,1,7,0,0,1,7,1,2,0,0,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,33,11,10,12,1,0,0,1,0.3333333333333333,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,115.45642705977977,91.17302234410326,67.51837839753202 +widget.cc,widget.cc,unit,1,10,0,0,0,0,5,2.5,1,1,5,2.5,1,1,9,19,6,9,28,44.039100017307746,1.5728250006181337,15,109.39293667703852,6.75,0.14814814814814814,738.40232257001,41.02235125388945,0.027231659910827242,10,9,2,0,1,2,1.8,0.4,0,0.2,1,10,0,0,1,9,1,2,0,1,2,0,0.4,0,2,0.4,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,7.6,0,12,1,0,0,1,0.2,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,108.13440061021349,80.81321081486506,63.23649158492017 +widget.cc,ns,namespace,2,10,0,0,0,0,4,2,1,1,4,2,1,1,9,19,6,9,28,44.039100017307746,1.5728250006181337,15,109.39293667703852,6.75,0.14814814814814814,738.40232257001,41.02235125388945,0.027231659910827242,9,9,2,0,0,2.25,2.25,0.5,0,0,1,9,0,0,1,9,1,2,0,0,2,0,0.5,0,2,0.5,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,9.5,5,12,1,0,0,1,0.25,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,110.0712409638703,83.50566092867489,64.36914676249725 +widget.cc,Widget,class,3,9,0,0,0,0,3,1.5,1,1,3,1.5,1,1,9,17,5,8,25,40.13896548741762,1.6055586194967049,14,95.1838730514401,7.2,0.1388888888888889,685.3238859703688,38.07354922057604,0.02591051177740593,7,7,2,0,0,2.3333333333333335,2.3333333333333335,0.6666666666666666,0,0,1,7,0,0,1,7,1,2,0,0,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,33,11,10,12,1,0,0,1,0.3333333333333333,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,115.09604084632235,90.65309494114345,67.3076262259195 widget.cc,value,function,5,5,0,0,0,0,1,1,1,1,1,1,1,1,5,5,2,2,7,13.60964047443681,1.944234353490973,7,19.651484454403228,2.5,0.4,49.12871113600807,2.7293728408893374,0.004471319041702271,1,1,1,0,0,1,1,1,0,0,1,1,0,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,10,10,10,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,155.2836050228807,148.4278547652622,90.80912574437467 widget.cc,set,function,6,6,0,0,0,0,1,1,1,1,1,1,1,1,6,6,3,4,10,20.264662506490403,2.0264662506490403,9,31.69925001442312,4,0.25,126.79700005769249,7.044277780982916,0.008413009854528818,1,1,1,0,0,1,1,1,0,0,1,1,0,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,12,12,12,12,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,152.79727628688778,144.8408406278255,89.35513233151332 diff --git a/tests/parity/cpp_mozcpp_parity.rs b/tests/parity/cpp_mozcpp_parity.rs index 39996719..73c9f03c 100644 --- a/tests/parity/cpp_mozcpp_parity.rs +++ b/tests/parity/cpp_mozcpp_parity.rs @@ -182,13 +182,14 @@ fn cpp_and_mozcpp_agree_on_this() { ); let get = |key: &str| metric(&cpp, key); - // Operators: `{` x3, `int` x3, `;` x4, `(` x2, `return` x2, `->` x2, - // and the `*` of `S* p` -> N1 = 17. `this` must not appear here — - // classifying it as an operator (the Java / C# / Kotlin reading, - // #1380) would take this to 18 and the operand row to 9. + // Operators: `struct` (#1552), `{` x3, `int` x3, `;` x4, `(` x2, + // `return` x2, `->` x2, and the `*` of `S* p` -> N1 = 18. `this` must + // not appear here — classifying it as an operator (the Java / C# / + // Kotlin reading, #1380) would take this to 19 and the operand row + // to 9. assert_eq!( get("halstead.operators"), - 17, + 18, "`this` must not be billed as an operator: {cpp:?}" ); // Operands: `S` x2, `x` x3, `m1`, `m2`, `p` x2, `this` -> N2 = 10. diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index 6b147c7a..6a4aaa59 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit 6b147c7a2431293ce44fa306b715fa6e1f31e279 +Subproject commit 6a4aaa595da0bf2a4b0c03fea3b186be06c92f83 From 079fd45d7f4cbc6ceb2c16b2574bb83f1c5a3fe5 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:05:54 -0700 Subject: [PATCH 29/48] fix(getter/js): bill a function expression once The JS-family operator arm listed both the `function` keyword leaf and the `function_expression` wrapper that contains it, so `const f = function () {}` billed `function` twice: n1 and N1 each one too high (grammar-dispatch section 5). The wrapper is dropped and the leaf kept. The leaf exists in every spelling: plain, `async`, and an ERROR-recovery parse, where the wrapper is absent. A generator expression is its own `generator_function` node and was never billed. JavaScript, Mozjs, TypeScript and TSX share the macro. The pdf.js corpus moves in halstead.* and mi.* only: 157 files, each changed space losing exactly one unique operator. Fixes #1554 --- big-code-analysis-ast/src/getter.rs | 15 ++++--- src/metrics/halstead.rs | 48 +++++++++++++++++++++ tests/repositories/big-code-analysis-output | 2 +- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index 90d77eaa..b1d471ec 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -246,12 +246,15 @@ macro_rules! impl_js_family_get_op_type { | STAREQ | SLASHEQ | PERCENTEQ | STARSTAREQ | GTGTEQ | GTGTGTEQ | LTLTEQ | AMPEQ | CARET | CARETEQ | PIPEEQ | Yield | LBRACK | LBRACE | Await | QMARK | QMARKQMARK | EQGT | DOTDOTDOT | New | Let | Var | Const | Function - | FunctionExpression | SEMI | Typeof | Instanceof | Void - // `Class2` is the `class` keyword leaf, billed beside - // `function` (#1552). The unsuffixed `Class` is the - // class *expression* node that wraps it - // (`const C = class {}`), so it stays unlisted and an - // expression bills one `class`, not two. + | SEMI | Typeof | Instanceof | Void + // `Function` and `Class2` are the `function` / `class` + // keyword leaves (#1552). Their expression wrappers — + // `FunctionExpression` and the unsuffixed `Class` — + // stay unlisted, so `const f = function () {}` bills + // one `function`, not two (#1554). The leaf is the + // keeper because it exists in every spelling: the + // `async` and generator forms, and an ERROR-recovery + // parse that drops the wrapper. | Class2 // `get`/`set` accessor keywords are operators, matching the // C# getter's `Get | Set | Init | Add | Remove` accessor arm. diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index e27154b1..af417cf4 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -6029,6 +6029,54 @@ end", assert_operator_rows::("foo.js", JS_FAMILY_CLASS_ROWS); } + /// #1554: a function *expression* bills its `function` keyword once, + /// as a declaration does — the `function_expression` wrapper bills + /// nothing of its own. Before #1554 the plain and `async` rows billed + /// one extra unique and total operator, and the nested row one extra + /// unique and two extra totals. The declaration and generator rows + /// are controls: neither node was ever in the operator arm. + #[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))] + const JS_FAMILY_FUNCTION_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `function`, `()`, `{}` + ("declaration", "function h() {}\n", [3, 3]), + // `const`, `=`, `function`, `()`, `{}`, `;` + ("expression", "const f = function () {};\n", [6, 6]), + // `const`, `=`, `async`, `function`, `()`, `{}`, `;` + ( + "async expression", + "const f = async function () {};\n", + [7, 7], + ), + // `const`, `=`, `function`, `*`, `()`, `{}`, `;` — a generator + // expression is its own `generator_function` node, never billed + ( + "generator expression", + "const f = function* () {};\n", + [7, 7], + ), + // `const`, `=`, `return`, `;` x2, and `function`, `()`, `{}` + // twice each + ( + "nested expressions", + "const f = function () { return function () {}; };\n", + [7, 11], + ), + ]; + + #[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))] + #[test] + fn js_family_function_expression_bills_function_once_1554() { + #[cfg(feature = "javascript")] + assert_operator_rows::("foo.js", JS_FAMILY_FUNCTION_ROWS); + #[cfg(feature = "mozjs")] + assert_operator_rows::("foo.js", JS_FAMILY_FUNCTION_ROWS); + #[cfg(feature = "typescript")] + { + assert_operator_rows::("foo.ts", JS_FAMILY_FUNCTION_ROWS); + assert_operator_rows::("foo.tsx", JS_FAMILY_FUNCTION_ROWS); + } + } + /// #1552 for the two TypeScript grammars: the shared `class` rows, /// plus their own `interface` and `enum` declaration keywords. #[cfg(feature = "typescript")] diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index 6a4aaa59..5d8271e0 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit 6a4aaa595da0bf2a4b0c03fea3b186be06c92f83 +Subproject commit 5d8271e0fb27535fce6c39b977ea72fae095956e From 011d4bf747961a4b51931730229c6db942638f54 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:18:56 -0700 Subject: [PATCH 30/48] fix(cpp): open no space for a bodiless specifier C++ and Mozcpp opened a struct / class space for every `struct_specifier` / `class_specifier`, including those that only name a type: forward declarations (`struct S;`), elaborated type specifiers (`struct S *p;`, casts, parameter types) and explicit template instantiations. Each spurious space sat at unit or function level, inflated cyclomatic.sum by its base of 1, and skewed every per-space min and average. A new shared predicate, `cpp_specifier_has_body`, gates both `is_func_space` and `get_space_kind` on the `body` field, so the space tree cannot disagree with itself (grammar-dispatch sections 6 and 7). Neither grammar overrides the `_with_code` forms, which forward to these. C and ObjC open no struct spaces and are unchanged. 105 DeepSpeech snapshots lose 4452 bodiless spaces (4207 struct, 245 class); no space is added, none removed had children, and every retained struct/class span has a body. Fixes #1555 --- big-code-analysis-ast/src/checker.rs | 13 +++ big-code-analysis-ast/src/checker/cpp.rs | 26 +++-- big-code-analysis-ast/src/checker/mozcpp.rs | 26 +++-- big-code-analysis-ast/src/getter/cpp.rs | 6 +- big-code-analysis-ast/src/getter/mozcpp.rs | 6 +- src/spaces_tests.rs | 119 ++++++++++++++++++++ tests/repositories/big-code-analysis-output | 2 +- 7 files changed, 171 insertions(+), 27 deletions(-) diff --git a/big-code-analysis-ast/src/checker.rs b/big-code-analysis-ast/src/checker.rs index 212f0d78..a17245ca 100644 --- a/big-code-analysis-ast/src/checker.rs +++ b/big-code-analysis-ast/src/checker.rs @@ -818,6 +818,19 @@ pub fn csharp_member_has_accessors(node: &Node) -> bool { csharp_accessor_count(node) > 0 } +/// Returns `true` when a C++ `struct_specifier` / `class_specifier` +/// defines a type body (`struct S { … }`) rather than only naming one. +/// A forward declaration (`struct S;`) and an elaborated type specifier +/// (`struct S *p;`, the parameter type in `void f(struct S *q)`) carry no +/// `body` field. +/// +/// Shared by the C++ and Mozcpp `is_func_space` and `get_space_kind`, so +/// a bodiless specifier neither opens a space nor is labelled one (#1555). +#[must_use] +pub(crate) fn cpp_specifier_has_body(node: &Node) -> bool { + node.child_by_field_name("body").is_some() +} + /// Whether `node` is a bare keyword token the C# grammar *aliased* to /// `modifier`, rather than the `modifier` rule itself. /// diff --git a/big-code-analysis-ast/src/checker/cpp.rs b/big-code-analysis-ast/src/checker/cpp.rs index 7c48d33e..4539bb24 100644 --- a/big-code-analysis-ast/src/checker/cpp.rs +++ b/big-code-analysis-ast/src/checker/cpp.rs @@ -21,18 +21,22 @@ impl Checker for CppCode { // will silently drop those nodes from FuncSpace creation the next // time a grammar bump starts emitting them (see lesson 2 in // `docs/development/lessons_learned.md`). + // + // A struct / class specifier opens a space only when it has a body, + // gated by the same predicate as `get_space_kind` (#1555). fn is_func_space(node: &Node) -> bool { - matches!( - node.kind_id().into(), - Cpp::TranslationUnit - | Cpp::FunctionDefinition - | Cpp::FunctionDefinition2 - | Cpp::FunctionDefinition3 - | Cpp::FunctionDefinition4 - | Cpp::StructSpecifier - | Cpp::ClassSpecifier - | Cpp::NamespaceDefinition - ) + match node.kind_id().into() { + Cpp::StructSpecifier | Cpp::ClassSpecifier => cpp_specifier_has_body(node), + kind => matches!( + kind, + Cpp::TranslationUnit + | Cpp::FunctionDefinition + | Cpp::FunctionDefinition2 + | Cpp::FunctionDefinition3 + | Cpp::FunctionDefinition4 + | Cpp::NamespaceDefinition + ), + } } // Issue #285 contract: keep this in sync with `is_func_space` and diff --git a/big-code-analysis-ast/src/checker/mozcpp.rs b/big-code-analysis-ast/src/checker/mozcpp.rs index 5d681996..7f46535d 100644 --- a/big-code-analysis-ast/src/checker/mozcpp.rs +++ b/big-code-analysis-ast/src/checker/mozcpp.rs @@ -21,18 +21,22 @@ impl Checker for MozcppCode { // will silently drop those nodes from FuncSpace creation the next // time a grammar bump starts emitting them (see lesson 2 in // `docs/development/lessons_learned.md`). + // + // A struct / class specifier opens a space only when it has a body, + // gated by the same predicate as `get_space_kind` (#1555). fn is_func_space(node: &Node) -> bool { - matches!( - node.kind_id().into(), - Mozcpp::TranslationUnit - | Mozcpp::FunctionDefinition - | Mozcpp::FunctionDefinition2 - | Mozcpp::FunctionDefinition3 - | Mozcpp::FunctionDefinition4 - | Mozcpp::StructSpecifier - | Mozcpp::ClassSpecifier - | Mozcpp::NamespaceDefinition - ) + match node.kind_id().into() { + Mozcpp::StructSpecifier | Mozcpp::ClassSpecifier => cpp_specifier_has_body(node), + kind => matches!( + kind, + Mozcpp::TranslationUnit + | Mozcpp::FunctionDefinition + | Mozcpp::FunctionDefinition2 + | Mozcpp::FunctionDefinition3 + | Mozcpp::FunctionDefinition4 + | Mozcpp::NamespaceDefinition + ), + } } // Issue #285 contract: keep this in sync with `is_func_space` and diff --git a/big-code-analysis-ast/src/getter/cpp.rs b/big-code-analysis-ast/src/getter/cpp.rs index 4841d746..37ecdd65 100644 --- a/big-code-analysis-ast/src/getter/cpp.rs +++ b/big-code-analysis-ast/src/getter/cpp.rs @@ -63,8 +63,10 @@ impl Getter for CppCode { match node.kind_id().into() { FunctionDefinition | FunctionDefinition2 | FunctionDefinition3 | FunctionDefinition4 => SpaceKind::Function, - StructSpecifier => SpaceKind::Struct, - ClassSpecifier => SpaceKind::Class, + // Bodiless specifiers open no space (#1555); see + // `cpp_specifier_has_body`. + StructSpecifier if crate::checker::cpp_specifier_has_body(node) => SpaceKind::Struct, + ClassSpecifier if crate::checker::cpp_specifier_has_body(node) => SpaceKind::Class, NamespaceDefinition => SpaceKind::Namespace, TranslationUnit => SpaceKind::Unit, _ => SpaceKind::Unknown, diff --git a/big-code-analysis-ast/src/getter/mozcpp.rs b/big-code-analysis-ast/src/getter/mozcpp.rs index a7a0e88c..ac9a0800 100644 --- a/big-code-analysis-ast/src/getter/mozcpp.rs +++ b/big-code-analysis-ast/src/getter/mozcpp.rs @@ -63,8 +63,10 @@ impl Getter for MozcppCode { match node.kind_id().into() { FunctionDefinition | FunctionDefinition2 | FunctionDefinition3 | FunctionDefinition4 => SpaceKind::Function, - StructSpecifier => SpaceKind::Struct, - ClassSpecifier => SpaceKind::Class, + // Bodiless specifiers open no space (#1555); see + // `cpp_specifier_has_body`. + StructSpecifier if crate::checker::cpp_specifier_has_body(node) => SpaceKind::Struct, + ClassSpecifier if crate::checker::cpp_specifier_has_body(node) => SpaceKind::Class, NamespaceDefinition => SpaceKind::Namespace, TranslationUnit => SpaceKind::Unit, _ => SpaceKind::Unknown, diff --git a/src/spaces_tests.rs b/src/spaces_tests.rs index 51cbbc3d..cee8f025 100644 --- a/src/spaces_tests.rs +++ b/src/spaces_tests.rs @@ -91,6 +91,125 @@ fn cpp_function_definition_is_classified_as_function() { ); } +/// #1555 fixture: five bodiless specifiers (two forward declarations, +/// an elaborated variable type, and two elaborated parameter types) +/// beside four bodied definitions, including the `final` / base-clause, +/// template and anonymous forms whose `body` follows optional preambles. +#[cfg(any(feature = "cpp", feature = "mozcpp"))] +const CPP_BODILESS_SPECIFIERS: &str = "struct S; +struct S *p; +class Fwd; +void f(struct S *q); +void g(class Fwd *r) {} +struct T {} t; +class C final : public Fwd { int x; }; +template struct X {}; +struct {} anon; +"; + +/// #1555: only a struct / class specifier with a body opens a space. The +/// bodiless ones used to open one each, at unit level even when they sat +/// in a parameter list. +#[cfg(any(feature = "cpp", feature = "mozcpp"))] +fn assert_only_bodied_specifiers_open_spaces() { + check_func_space::(CPP_BODILESS_SPECIFIERS, "fwd.cpp", |root| { + let top: Vec<(SpaceKind, Option<&str>, usize)> = root + .spaces + .iter() + .map(|s| (s.kind, s.name.as_deref(), s.start_line)) + .collect(); + assert_eq!( + top, + vec![ + (SpaceKind::Function, Some("g"), 5), + (SpaceKind::Struct, Some("T"), 6), + (SpaceKind::Class, Some("C"), 7), + (SpaceKind::Struct, Some("X"), 8), + (SpaceKind::Struct, None, 9), + ], + "top-level spaces (kind, name, line); got {top:?}" + ); + assert!( + root.spaces.iter().all(|s| s.spaces.is_empty()), + "no space nests below these: `g`'s `class Fwd` parameter type opens none" + ); + }); +} + +#[cfg(feature = "cpp")] +#[test] +fn cpp_bodiless_specifiers_open_no_space_1555() { + assert_only_bodied_specifiers_open_spaces::(); +} + +#[cfg(feature = "mozcpp")] +#[test] +fn mozcpp_bodiless_specifiers_open_no_space_1555() { + assert_only_bodied_specifiers_open_spaces::(); +} + +/// #1555: `is_func_space` and `get_space_kind` agree on every struct / +/// class specifier node, checked node by node because the space tree +/// alone cannot see the getter half — it only labels spaces the checker +/// already opened. The oracle is independent of the `body` field: a +/// specifier defines a type exactly when its text has a `{`. +#[cfg(any(feature = "cpp", feature = "mozcpp"))] +fn assert_specifier_predicates_agree(struct_id: u16, class_id: u16) { + use crate::checker::Checker; + use crate::getter::Getter; + + let parser = T::new( + CPP_BODILESS_SPECIFIERS.as_bytes().to_vec(), + &std::path::PathBuf::from("fwd.cpp"), + None, + ); + let code = CPP_BODILESS_SPECIFIERS.as_bytes(); + let (mut bodied, mut bodiless) = (0, 0); + for node in parser.root().preorder() { + let id = node.kind_id(); + let kind = if id == struct_id { + SpaceKind::Struct + } else if id == class_id { + SpaceKind::Class + } else { + continue; + }; + let has_body = code[node.start_byte()..node.end_byte()].contains(&b'{'); + if has_body { + bodied += 1; + } else { + bodiless += 1; + } + assert_eq!(T::Checker::is_func_space(&node), has_body, "{node:?}"); + assert_eq!( + T::Getter::get_space_kind(&node), + if has_body { kind } else { SpaceKind::Unknown }, + "{node:?}" + ); + } + assert_eq!((bodied, bodiless), (4, 5), "fixture specifier census"); +} + +#[cfg(feature = "cpp")] +#[test] +fn cpp_specifier_space_predicates_agree_1555() { + use crate::Cpp; + assert_specifier_predicates_agree::( + Cpp::StructSpecifier as u16, + Cpp::ClassSpecifier as u16, + ); +} + +#[cfg(feature = "mozcpp")] +#[test] +fn mozcpp_specifier_space_predicates_agree_1555() { + use crate::Mozcpp; + assert_specifier_predicates_agree::( + Mozcpp::StructSpecifier as u16, + Mozcpp::ClassSpecifier as u16, + ); +} + #[cfg(feature = "cpp")] #[test] fn cpp_scope_resolution_operator() { diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index 5d8271e0..44c17e00 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit 5d8271e0fb27535fce6c39b977ea72fae095956e +Subproject commit 44c17e002c1850b0a4e6bac32b66a2b66113c417 From 852fdc8c331c23c3cbc7ec862d624ab8addd3752 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:28:31 -0700 Subject: [PATCH 31/48] fix(getter/kotlin): bill the interface keyword Kotlin billed `class` and `object` but not `interface`, while Java, C#, TypeScript and Groovy bill theirs (#1552). The `interface` leaf is now an operator; its `class_declaration` wrapper stays unlisted, so a declaration bills its keyword once. The grammar has one `interface` kind, so there is no alias to cover. `enum` in `enum class` stays unbilled. Kotlin's grammar parses it as a `class_modifier`, the same node that holds `data`, `sealed` and `annotation`, and the getter bills none of those, nor `abstract` or any visibility modifier. The declaration keyword is the `class` that follows, which already bills, so `enum class E` scores one keyword just as Java's `enum E` does. Billing `enum` alone would single out one member of a modifier class the getter otherwise ignores; whether Kotlin should bill modifiers at all, as Java and C# do, is a separate question. No Kotlin corpus is snapshotted, so no integration snapshot moves. Fixes #1556 --- big-code-analysis-ast/src/getter/kotlin.rs | 11 +++++--- big-code-analysis-book/src/metrics.md | 4 ++- src/metrics/halstead.rs | 30 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index 7c2bad61..e0ba1dc1 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -217,9 +217,14 @@ impl Getter for KotlinCode { // Operator: control flow keywords If | Else | When | For | While | Do | Try | Catch | Finally | Throw | Return | ReturnAT - // Operator: other keywords - | Class | Fun | Object | Val | Var | In | Is | As | AsQMARK | BANGis | BANGin - | Constructor + // Operator: other keywords. `interface` is the declaration + // keyword leaf beside `class` and `object` (#1556); its + // `class_declaration` wrapper stays unlisted. `enum` in + // `enum class` is not here: the grammar makes it a + // `class_modifier`, the same node as `data`, `sealed` and + // `annotation`, and Kotlin bills no modifier. + | Class | Interface | Fun | Object | Val | Var | In | Is | As | AsQMARK | BANGis + | BANGin | Constructor // Operator: brackets, separators, terminators | SEMI | COMMA | COLONCOLON | DOT | LBRACE | LBRACK | LPAREN // Operator: assignment and arithmetic diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index e5f73b85..f4e17212 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -717,7 +717,9 @@ tokens involved the same way they spell real operators: `interface`, `record`, `trait`, `@interface`, `@implementation` and `@protocol`. A `class` that declares no class is not billed: Java's `C.class` stays one operand, and C++'s `template ` scores - like `template `. + like `template `. Kotlin's `interface` followed in #1556; + the `enum` of `enum class` is a modifier there, like `data` and + `sealed`, and Kotlin bills no modifier. - **An alternative spelling is an operator of its own.** C++ spells `&&` equally as `and`, `!=` as `not_eq` and so on through all eleven [alternative tokens](https://en.cppreference.com/w/cpp/language/operator_alternative). diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index af417cf4..352d0108 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -6235,6 +6235,36 @@ end", assert_operator_rows::("foo.m", ROWS); } + /// #1556: Kotlin's `interface` bills as its `class` and `object` + /// do. The modifier rows pin the other half of the decision: `enum` + /// in `enum class` is a `class_modifier` like `data`, and neither + /// bills, so the three modifier rows score exactly as `class` does. + #[cfg(feature = "kotlin")] + #[test] + fn kotlin_declaration_keywords_are_operators_1556() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}` + ("class", "class C {}\n", [2, 2]), + // `interface`, `{}` + ("interface", "interface I {}\n", [2, 2]), + // `object`, `{}` + ("object", "object O {}\n", [2, 2]), + // `class`, `{}` — no `enum` + ("enum class", "enum class E { A }\n", [2, 2]), + // `class`, `{}` — no `data` + ("data class", "data class D {}\n", [2, 2]), + // `interface`, `{}` — no `sealed` + ("sealed interface", "sealed interface S {}\n", [2, 2]), + // `interface`, `class`, `:`, `{}` twice + ( + "implementing class", + "interface J {}\nclass K : J {}\n", + [4, 5], + ), + ]; + assert_operator_rows::("foo.kt", ROWS); + } + #[cfg(feature = "bash")] #[test] fn bash_operators_and_operands() { From 86ff0343dac0c297cd37598eae3d6975ae38ad3e Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:48:00 -0700 Subject: [PATCH 32/48] fix(getter): bill C++ and TS declarations and names TypeScript billed no operand for a type name. `type_identifier` is the one leaf kind for every type name: a class, interface or alias name, a type parameter, an annotation (`x: Foo`), a generic (`Map`), an `implements` entry, the tail of `ns.T`. It was in neither arm, so `interface I {}` and `type T = number` lost their names, and TS read `class C {}` with one operand fewer than JavaScript. C#, Rust, C, C++, ObjC, Go and Groovy bill type names as operands; `TypeIdentifier` is now a TS and TSX operand extra. No classified node wraps it (`generic_type` and `nested_type_identifier` are unlisted), so each name bills once. Java has the same gap, filed as #1560. The declaration keywords #1552 left out are now operators, each as its keyword leaf under an unclassified wrapper: - C++ / Mozcpp: `namespace`, `template`, `typename`, `typedef`, `using`. They also bill in their non-declaring positions (`using namespace`, `typename T::x`, `x.template f`), where they are the same leaf. - C and ObjC: `typedef`, so a header scores alike whichever grammar parses it. - TypeScript / TSX: `type`, `namespace`, `module`, `declare`, and `global` (the `declare global {}` sibling of `declare module`). `Module2` is the keyword leaf; the `Module` declaration node stays unlisted. Each word is contextual, but used as a name the grammar emits an `identifier`, so no parent gate is needed. #1552 left C++'s template-parameter `class` unbilled because its synonym `typename` was. With `typename` billed the guard is gone, and `template ` and `template ` still score alike. Template `<` `>` stay two operators: every language with generics bills them that way, so the pair question is filed as #1559. Snapshots: 929 DeepSpeech C++ files move in halstead.* and mi.* only (field-level diff); pdf.js and serde do not move. Submodule 58a78b26. Fixes #1557 --- big-code-analysis-ast/src/getter/c.rs | 11 +- big-code-analysis-ast/src/getter/cpp.rs | 31 +-- big-code-analysis-ast/src/getter/mozcpp.rs | 19 +- big-code-analysis-ast/src/getter/objc.rs | 11 +- big-code-analysis-ast/src/getter/tsx.rs | 10 +- .../src/getter/typescript.rs | 41 +++- big-code-analysis-book/src/metrics.md | 15 +- src/metrics/halstead.rs | 225 +++++++++++++++--- ...put_formats__csv_test__csv_cpp_widget.snap | 4 +- tests/repositories/big-code-analysis-output | 2 +- 10 files changed, 263 insertions(+), 106 deletions(-) diff --git a/big-code-analysis-ast/src/getter/c.rs b/big-code-analysis-ast/src/getter/c.rs index 9aa3bd22..8fb6520b 100644 --- a/big-code-analysis-ast/src/getter/c.rs +++ b/big-code-analysis-ast/src/getter/c.rs @@ -81,11 +81,12 @@ impl Getter for CCode { // `operators` store) keeps them distinct in n1 while `long long`'s // two `long` tokens correctly fold to one n1 entry but two N1 hits. | Signed | Unsigned | Long | Short - // `struct` / `union` / `enum`, which C++ bills since #1552. - // They are C keywords, not C++-only forms, so leaving them - // out here would score a header differently by the grammar - // that parsed it. The derivation is on the Cpp arm. - | Struct | Union | Enum => TokenRole::Operator, + // `struct` / `union` / `enum`, which C++ bills since #1552, + // and `typedef` since #1557. They are C keywords, not + // C++-only forms, so leaving them out here would score a + // header differently by the grammar that parsed it. The + // derivation is on the Cpp arm. + | Struct | Union | Enum | Typedef => TokenRole::Operator, // `CharLiteral` joins the operand list here for the whole // C family — `cpp.rs`, `mozcpp.rs` and `objc.rs` carry the // same arm and point back at this note (#1316). Before it a diff --git a/big-code-analysis-ast/src/getter/cpp.rs b/big-code-analysis-ast/src/getter/cpp.rs index 37ecdd65..b73b611a 100644 --- a/big-code-analysis-ast/src/getter/cpp.rs +++ b/big-code-analysis-ast/src/getter/cpp.rs @@ -111,25 +111,6 @@ impl Getter for CppCode { { TokenRole::Unknown } - // `class` in a template parameter list (`template `, - // `class U = int`, `class... Ts`) declares a type parameter, - // not a class: it is the exact synonym of `typename`, which - // is unbilled, so it stays unbilled too and the two - // spellings score alike. Everywhere else — a class - // specifier, `enum class`, a forward declaration — it bills - // as the declaration keyword it is (#1552). - Class - if matches!( - ancestors.parent(node).map(|p| p.kind_id().into()), - Some( - TypeParameterDeclaration - | OptionalTypeParameterDeclaration - | VariadicTypeParameterDeclaration - ) - ) => - { - TokenRole::Unknown - } DOT | DOTSTAR | LPAREN | LPAREN2 | COMMA | STAR | GTGT | COLON | SEMI | Return | Break | Continue | If | Else | Switch | Case | Default | For | While | Goto | Do | Delete | New | Try | Try2 | Catch | Throw | EQ | AMPAMP | PIPEPIPE | DASH @@ -194,7 +175,17 @@ impl Getter for CppCode { // keyword once — and `enum class` bills both of its two. // The same leaf heads an elaborated type (`struct S s;`), // where it is billed too, as `int` is in `int s;`. - | Class | Struct | Union | Enum + // + // #1557 adds the remaining declaration keywords: `namespace`, + // `template`, `typename`, `typedef` and `using`, each the + // leaf of an unclassified wrapper (`namespace_definition`, + // `template_declaration`, `type_parameter_declaration`, + // `type_definition`, `alias_declaration` / + // `using_declaration`). A template-parameter `class` bills + // with them: it is the synonym of `typename`, so the two + // spellings of `template ` still score alike, now + // both billed. + | Class | Struct | Union | Enum | Namespace | Template | Typename | Typedef | Using => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only diff --git a/big-code-analysis-ast/src/getter/mozcpp.rs b/big-code-analysis-ast/src/getter/mozcpp.rs index ac9a0800..eab281af 100644 --- a/big-code-analysis-ast/src/getter/mozcpp.rs +++ b/big-code-analysis-ast/src/getter/mozcpp.rs @@ -90,21 +90,6 @@ impl Getter for MozcppCode { { TokenRole::Unknown } - // `class` in a template parameter list is `typename`'s - // synonym and stays unbilled — the derivation is on the Cpp - // twin (#1552). - Class - if matches!( - ancestors.parent(node).map(|p| p.kind_id().into()), - Some( - TypeParameterDeclaration - | OptionalTypeParameterDeclaration - | VariadicTypeParameterDeclaration - ) - ) => - { - TokenRole::Unknown - } DOT | DOTSTAR | LPAREN | LPAREN2 | COMMA | STAR | GTGT | COLON | SEMI | Return | Break | Continue | If | Else | Switch | Case | Default | For | While | Goto | Do | Delete | New | Try | Try2 | Catch | Throw | EQ | AMPAMP | PIPEPIPE | DASH @@ -134,8 +119,8 @@ impl Getter for MozcppCode { // derivation is on the Cpp twin (#1547). | CoReturn | CoYield // The type-declaration keywords — the derivation is on the - // Cpp twin (#1552). - | Class | Struct | Union | Enum + // Cpp twin (#1552, #1557). + | Class | Struct | Union | Enum | Namespace | Template | Typename | Typedef | Using => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only diff --git a/big-code-analysis-ast/src/getter/objc.rs b/big-code-analysis-ast/src/getter/objc.rs index ead1313c..1960fce7 100644 --- a/big-code-analysis-ast/src/getter/objc.rs +++ b/big-code-analysis-ast/src/getter/objc.rs @@ -95,9 +95,10 @@ impl Getter for ObjcCode { // keywords / markers from `In` onwards. // // From `Struct` onwards come the type-declaration keywords - // (#1552): C's `struct` / `union` / `enum` (see `c.rs`), and - // ObjC's own class and protocol declarations. Each is the - // leaf of a `class_interface` / `class_implementation` / + // (#1552): C's `struct` / `union` / `enum` and, since #1557, + // `typedef` (see `c.rs`), and ObjC's own class and protocol + // declarations. Each is the leaf of a `type_definition` / + // `class_interface` / `class_implementation` / // `protocol_declaration`, none of which is classified, so a // declaration bills its keyword once. `@class Fwd;` needs no // arm: it parses as an `@` token, already billed here, and a @@ -111,7 +112,9 @@ impl Getter for ObjcCode { | CARETEQ | PIPEEQ | LBRACK | LBRACE | QMARK | PrimitiveType | TypeSpecifier | Sizeof | Signed | Unsigned | Long | Short | In | AT | ATtry | ATcatch | ATfinally | ATthrow | ATsynchronized | ATautoreleasepool | ATselector | ATencode | Struct - | Union | Enum | ATinterface | ATimplementation | ATprotocol => TokenRole::Operator, + | Union | Enum | Typedef | ATinterface | ATimplementation | ATprotocol => { + TokenRole::Operator + } // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only // classified node in a character literal, so it bills one diff --git a/big-code-analysis-ast/src/getter/tsx.rs b/big-code-analysis-ast/src/getter/tsx.rs index 13a2b6ad..d50967c3 100644 --- a/big-code-analysis-ast/src/getter/tsx.rs +++ b/big-code-analysis-ast/src/getter/tsx.rs @@ -78,10 +78,16 @@ impl Getter for TsxCode { // `Interface` / `Enum` are the TS-only type-declaration keyword // leaves, billed as the shared `class` is (#1552) and as C# and // Java bill theirs; their `*_declaration` wrappers stay unlisted. + // + // `Type` / `Namespace` / `Module2` / `Declare` / `Global` and the + // `TypeIdentifier` operand: see the TS invocation (#1557). impl_js_family_get_op_type!( Tsx, - op_extras: [QMARKDOT, PredefinedType, Interface, Enum, LTSLASH, SLASHGT], - operand_extras: [Identifier2, String2], + op_extras: [ + QMARKDOT, PredefinedType, Interface, Enum, Type, Namespace, Module2, Declare, Global, + LTSLASH, SLASHGT, + ], + operand_extras: [Identifier2, String2, TypeIdentifier], predefined_void: PredefinedType, jsx: [JsxText, HtmlCharacterReference], ); diff --git a/big-code-analysis-ast/src/getter/typescript.rs b/big-code-analysis-ast/src/getter/typescript.rs index 210b4e6b..d56a0a7e 100644 --- a/big-code-analysis-ast/src/getter/typescript.rs +++ b/big-code-analysis-ast/src/getter/typescript.rs @@ -52,15 +52,15 @@ impl Getter for TypescriptCode { bound_name.map_or(Some(""), |name| node_text(code, &name)) } - // TypeScript's operand extras are empty. `NestedIdentifier` and - // `MemberExpression4` — the TS-only member-expression productions — - // were listed until #1263 and are now deliberately absent, matching - // the `MemberExpression*` drop in the macro body: `namespace N.M` - // contributes the operands `N` and `M` plus the `.` operator, and - // `a.b` contributes `a` and `b`, never the composite text as well. - // The composite was billed on top of leaves the walker already - // reached, which is grammar-dispatch section 5's container/leaf - // double-count. + // TypeScript's only operand extra is `TypeIdentifier` (#1557, + // below). `NestedIdentifier` and `MemberExpression4` — the TS-only + // member-expression productions — were listed until #1263 and are + // now deliberately absent, matching the `MemberExpression*` drop in + // the macro body: `namespace N.M` contributes the operands `N` and + // `M` plus the `.` operator, and `a.b` contributes `a` and `b`, + // never the composite text as well. The composite was billed on top + // of leaves the walker already reached, which is grammar-dispatch + // section 5's container/leaf double-count. // // TS's anonymous `"string"` alias `String2` (kind_id 135, the // `: string` type keyword, emitted only as the child of a @@ -79,10 +79,29 @@ impl Getter for TypescriptCode { // `Interface` / `Enum` are the TS-only type-declaration keyword // leaves, billed as the shared `class` is (#1552) and as C# and // Java bill theirs; their `*_declaration` wrappers stay unlisted. + // + // #1557 adds the rest: `type`, `namespace`, `module`, `declare` and + // `global`. `Module2` is the `module` keyword leaf; the unsuffixed + // `Module` is the `module` declaration node that wraps it, so it + // stays unlisted. Each word is contextual, but used as a name + // (`let type = 1`, `module.exports`) the grammar emits an + // `identifier`, never the keyword leaf, so no parent gate is + // needed. `type` also bills in `import type` / `export type`, + // where it is the same leaf. + // + // `TypeIdentifier` is the operand extra: every type *name* — a + // class, interface or alias name, a type parameter, an annotation + // (`x: Foo`), a generic (`Map`), the tail of `ns.T` — is this + // one leaf kind, and it was in neither arm before #1557, so TS lost + // every one while C#, Rust and the C family bill theirs. No + // classified node wraps it (`generic_type` and + // `nested_type_identifier` are unlisted), so it bills once. impl_js_family_get_op_type!( Typescript, - op_extras: [QMARKDOT, PredefinedType, Interface, Enum], - operand_extras: [], + op_extras: [ + QMARKDOT, PredefinedType, Interface, Enum, Type, Namespace, Module2, Declare, Global, + ], + operand_extras: [TypeIdentifier], predefined_void: PredefinedType, ); diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index f4e17212..0aba4b98 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -715,11 +715,16 @@ tokens involved the same way they spell real operators: the JavaScript family, C, C++ and Objective-C followed in #1552 with their type-declaration keywords: `class`, `struct`, `union`, `enum`, `interface`, `record`, `trait`, `@interface`, `@implementation` and - `@protocol`. A `class` that declares no class is not billed: Java's - `C.class` stays one operand, and C++'s `template ` scores - like `template `. Kotlin's `interface` followed in #1556; - the `enum` of `enum class` is a modifier there, like `data` and - `sealed`, and Kotlin bills no modifier. + `@protocol`. Java's `C.class` is not billed: it stays one operand. + Kotlin's `interface` followed in #1556; the `enum` of `enum class` is + a modifier there, like `data` and `sealed`, and Kotlin bills no + modifier. #1557 added C++'s `namespace`, `template`, `typename`, + `typedef` and `using` (C and Objective-C bill `typedef` too), so + `template ` and `template ` each bill their + keyword and score alike. It also added TypeScript's `type`, + `namespace`, `module`, `declare` and `global`, and made every + TypeScript type name an operand (`x: Foo`, `interface I`, + `Map`), as C#, Rust and the C family already did. - **An alternative spelling is an operator of its own.** C++ spells `&&` equally as `and`, `!=` as `not_eq` and so on through all eleven [alternative tokens](https://en.cppreference.com/w/cpp/language/operator_alternative). diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 352d0108..0323c0aa 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -2300,32 +2300,25 @@ mod tests { // `;`×2, `(`, `return`, `.` — 9 total, 7 unique. (`class` joined // the JS-family operator arm in #1552.) // * Operands: `C`, `#x`, `1`, `m`, `this`, `#x` — 6 total, 5 - // unique under JS/MozJS. Under TS/TSX the class *name* `C` - // parses as `type_identifier`, which those getters do not - // classify, so both counts drop by one to 5/4 — a pre-existing - // divergence this fixture records rather than fixes. + // unique in all four grammars. Under TS/TSX the class *name* `C` + // is a `type_identifier`, which those getters did not classify + // until #1557, so they read 5/4 before it. #[cfg(all(feature = "javascript", feature = "mozjs", feature = "typescript"))] #[test] fn js_family_private_field_leaf_is_the_operand_1263() { #[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))] const SRC: &str = "class C { #x = 1; m() { return this.#x; } }"; - let check_js = |m: crate::CodeMetrics| { + let check = |m: crate::CodeMetrics| { assert_eq!(m.halstead.unique_operators(), 7); assert_eq!(m.halstead.total_operators(), 9); assert_eq!(m.halstead.unique_operands(), 5); assert_eq!(m.halstead.total_operands(), 6); }; - let check_ts = |m: crate::CodeMetrics| { - assert_eq!(m.halstead.unique_operators(), 7); - assert_eq!(m.halstead.total_operators(), 9); - assert_eq!(m.halstead.unique_operands(), 4); - assert_eq!(m.halstead.total_operands(), 5); - }; - check_metrics::(SRC, "foo.js", check_js); - check_metrics::(SRC, "foo.js", check_js); - check_metrics::(SRC, "foo.ts", check_ts); - check_metrics::(SRC, "foo.tsx", check_ts); + check_metrics::(SRC, "foo.js", check); + check_metrics::(SRC, "foo.js", check); + check_metrics::(SRC, "foo.ts", check); + check_metrics::(SRC, "foo.tsx", check); } // Issue #1263, the other section 6 half: `meta_property` is the one @@ -2362,8 +2355,8 @@ mod tests { // // expected, for `namespace N.M { }`: // - // * Operators: `.`, `{` — 2 total, 2 unique. (`namespace` is not in - // the JS-family operator arm.) + // * Operators: `namespace`, `.`, `{` — 3 total, 3 unique. + // (`namespace` joined the TS operator extras in #1557.) // * Operands: `N`, `M` — 2 total, 2 unique. Before the fix the // `nested_identifier` added `N.M`, making both 3. #[cfg(feature = "typescript")] @@ -2371,8 +2364,8 @@ mod tests { fn ts_nested_identifier_counts_leaves_not_the_composite_1263() { const SRC: &str = "namespace N.M { }"; let check = |m: crate::CodeMetrics| { - assert_eq!(m.halstead.unique_operators(), 2); - assert_eq!(m.halstead.total_operators(), 2); + assert_eq!(m.halstead.unique_operators(), 3); + assert_eq!(m.halstead.total_operators(), 3); assert_eq!(m.halstead.unique_operands(), 2); assert_eq!(m.halstead.total_operands(), 2); }; @@ -5878,14 +5871,15 @@ end", ); } - /// Asserts `[n1, N1]` for each `(label, source, expected)` row, one - /// parse per row so a row's counts are its own. + /// Asserts `count(halstead)` for each `(label, source, expected)` + /// row, one parse per row so a row's counts are its own. #[cfg(any( feature = "c", feature = "cpp", feature = "groovy", feature = "java", feature = "javascript", + feature = "kotlin", feature = "mozcpp", feature = "mozjs", feature = "objc", @@ -5893,16 +5887,19 @@ end", feature = "rust", feature = "typescript" ))] - fn assert_operator_rows(file: &str, rows: &[(&str, &str, [u64; 2])]) { + fn assert_halstead_rows( + file: &str, + rows: &[(&str, &str, [u64; 2])], + count: fn(&Stats) -> [u64; 2], + ) { for (label, source, expected) in rows { crate::test_support::check_func_space_only::( source, file, &[crate::Metric::Halstead], |space| { - let halstead = &space.metrics.halstead; assert_eq!( - [halstead.unique_operators(), halstead.total_operators()], + count(&space.metrics.halstead), *expected, "{label}: {source:?}" ); @@ -5911,6 +5908,31 @@ end", } } + /// Asserts `[n1, N1]` for each row. + #[cfg(any( + feature = "c", + feature = "cpp", + feature = "groovy", + feature = "java", + feature = "javascript", + feature = "kotlin", + feature = "mozcpp", + feature = "mozjs", + feature = "objc", + feature = "python", + feature = "rust", + feature = "typescript" + ))] + fn assert_operator_rows(file: &str, rows: &[(&str, &str, [u64; 2])]) { + assert_halstead_rows::(file, rows, |h| [h.unique_operators(), h.total_operators()]); + } + + /// Asserts `[n2, N2]` for each row. + #[cfg(feature = "typescript")] + fn assert_operand_rows(file: &str, rows: &[(&str, &str, [u64; 2])]) { + assert_halstead_rows::(file, rows, |h| [h.unique_operands(), h.total_operands()]); + } + /// #1486: Python's bracket openers, `;` and `:` are vocabulary, as /// they are in every other getter (#1395). Each row isolates one /// position, so the expected `[n1, N1]` is derivable from the row @@ -6146,8 +6168,9 @@ end", } /// #1552 rows shared by the two C++ clones. The template-parameter - /// rows pin the guard: `class T` is `typename T`'s synonym, and the - /// two spellings score alike. + /// rows pin the synonym: `class T` is `typename T`, and the two + /// spellings score alike. Both were unbilled until #1557 billed + /// `typename`, and the parameter `class` with it. #[cfg(any(feature = "cpp", feature = "mozcpp"))] const CPP_DECLARATION_KEYWORD_ROWS: &[(&str, &str, [u64; 2])] = &[ // `class`, `{}`, `;` @@ -6162,28 +6185,30 @@ end", ("enum class", "enum class F { B };\n", [4, 4]), // `struct`, `;` — an elaborated type bills its keyword ("elaborated type", "struct S s;\n", [2, 2]), - // `<`, `>`, `struct`, `{}`, `;` — no `class` + // `template`, `<`, `class`, `>`, `struct`, `{}`, `;` ( "class parameter", "template struct W {};\n", - [5, 5], + [7, 7], ), + // as above, `typename` in place of `class` ( "typename parameter", "template struct W {};\n", - [5, 5], + [7, 7], ), - // `<`, `=`, `int`, `>`, `struct`, `{}`, `;` + // `template`, `<`, `class`, `=`, `int`, `>`, `struct`, `{}`, `;` ( "optional class parameter", "template struct W {};\n", - [7, 7], + [9, 9], ), - // `<`, `>`, `struct`, `{}`, `;` — `...` is an operand + // `template`, `<`, `class`, `>`, `struct`, `{}`, `;` — `...` is + // an operand ( "variadic class parameter", "template struct W {};\n", - [5, 5], + [7, 7], ), ]; @@ -6265,6 +6290,128 @@ end", assert_operator_rows::("foo.kt", ROWS); } + /// #1557 rows shared by the two C++ clones: the declaration keywords + /// #1552 left out, one per row, plus the two disambiguator positions + /// (`typename T::x`, `x.template f`) that bill the same leaf. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const CPP_MORE_DECLARATION_KEYWORD_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `namespace`, `{}` + ("namespace", "namespace N {}\n", [2, 2]), + // `typedef`, `int`, `;` + ("typedef", "typedef int T;\n", [3, 3]), + // `using`, `=`, `int`, `;` + ("alias declaration", "using U = int;\n", [4, 4]), + // `using`, `namespace`, `;` + ("using directive", "using namespace std;\n", [3, 3]), + // `template`, `<`, `typename`, `>`, `;` and the `struct` of `S` + ( + "template declaration", + "template struct S;\n", + [6, 6], + ), + // `typename`, `::`, `;` + ("dependent type", "typename T::type x;\n", [3, 3]), + // `void`, `()` twice, `{}`, `.`, `template`, `<`, `int`, `>`, `;` + ( + "template disambiguator", + "void g() { x.template f(); }\n", + [9, 10], + ), + ]; + + /// One call per clone, as for the #1552 rows. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_more_declaration_keywords_are_operators_1557() { + assert_operator_rows::("foo.cpp", CPP_MORE_DECLARATION_KEYWORD_ROWS); + assert_operator_rows::("foo.cpp", CPP_MORE_DECLARATION_KEYWORD_ROWS); + } + + /// #1557: `typedef` is a C keyword, so C and ObjC bill it as C++ + /// does — a header scores alike whichever grammar parses it. + #[cfg(any(feature = "c", feature = "objc"))] + const C_TYPEDEF_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `typedef`, `int`, `;` + ("typedef", "typedef int T;\n", [3, 3]), + // `typedef`, `struct`, `{}`, `int`, `;` twice + ("typedef struct", "typedef struct S { int a; } S;\n", [5, 6]), + ]; + + #[cfg(any(feature = "c", feature = "objc"))] + #[test] + fn c_family_typedef_is_an_operator_1557() { + #[cfg(feature = "c")] + assert_operator_rows::("foo.c", C_TYPEDEF_ROWS); + #[cfg(feature = "objc")] + assert_operator_rows::("foo.m", C_TYPEDEF_ROWS); + } + + /// #1557: the TypeScript declaration keywords #1552 left out. The + /// last row pins that each word, used as a *name*, stays an operand: + /// the grammar emits an `identifier` there, never the keyword leaf. + #[cfg(feature = "typescript")] + const TS_DECLARATION_KEYWORD_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `type`, `=`, `number`, `;` + ("type alias", "type T = number;\n", [4, 4]), + // `namespace`, `{}` + ("namespace", "namespace N {}\n", [2, 2]), + // `module`, `{}`. Billing the `module` declaration node instead + // of its keyword leaf would score this row the same: every + // spelling probed (bodiless, quoted, dotted, exported, ERROR + // recovery) emits the two together, so only the getter's + // choice of the leaf, not this row, keeps them from both billing. + ("module", "module M {}\n", [2, 2]), + // `declare`, `module`, `{}` + ("ambient module", "declare module \"m\" {}\n", [3, 3]), + // `declare`, `global`, `{}` + ("global augmentation", "declare global {}\n", [3, 3]), + // `import`, `type`, `{}`, `from`, `;` + ( + "type-only import", + "import type { A } from \"a\";\n", + [5, 5], + ), + // `let`, `=` twice, `;` twice, `.` — no keyword + ( + "keywords as names", + "let type = 1;\nmodule.exports = declare;\n", + [4, 6], + ), + ]; + + /// #1557: a TypeScript type name is an operand, as C#, Rust and the + /// C family bill theirs. Each row's names are `type_identifier` + /// leaves except where noted, and before #1557 none of them billed. + #[cfg(feature = "typescript")] + const TS_TYPE_NAME_OPERAND_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `T` + ("alias name", "type T = number;\n", [1, 1]), + // `I` + ("interface name", "interface I {}\n", [1, 1]), + // `C` + ("class name", "class C {}\n", [1, 1]), + // `x` (an identifier), `Foo` + ("annotation", "let x: Foo;\n", [2, 2]), + // `m`, `Map`, `K` twice + ("generic", "let m: Map;\n", [3, 4]), + // `f`, `a` (identifiers), `U` twice + ("type parameter", "function f(a: U) {}\n", [3, 4]), + // `q`, `ns` (identifiers), `T` — the `nested_type_identifier` + // wrapper is unlisted, so its text is not a fourth operand + ("qualified type", "let q: ns.T;\n", [3, 3]), + // `C`, `I` + ("implements", "class C implements I {}\n", [2, 2]), + ]; + + #[cfg(feature = "typescript")] + #[test] + fn typescript_declaration_keywords_and_type_names_1557() { + assert_operator_rows::("foo.ts", TS_DECLARATION_KEYWORD_ROWS); + assert_operator_rows::("foo.tsx", TS_DECLARATION_KEYWORD_ROWS); + assert_operand_rows::("foo.ts", TS_TYPE_NAME_OPERAND_ROWS); + assert_operand_rows::("foo.tsx", TS_TYPE_NAME_OPERAND_ROWS); + } + #[cfg(feature = "bash")] #[test] fn bash_operators_and_operands() { @@ -10754,13 +10901,13 @@ S s; int a = s[1]; auto t = co_await s; // `kenlm/lm/search_trie.cc:286`: a `typename` functional cast // used as a statement. Recovery inserts a MISSING `)` after - // `quant_` and a MISSING `;` after `order`. Operators (n1 = 8, - // N1 = 10): `void`, `()` 3, `{}`, `::`, `,`, `-`, `.`, the one - // written `;`. Operands (n2 = 7, N2 = 7): `f`, `MiddlePointer`, - // `quant_`, `order`, `2`, `Write`, `w`. Before #1546 the MISSING - // `;` was a second `;`: (8, 11, 7, 7). + // `quant_` and a MISSING `;` after `order`. Operators (n1 = 9, + // N1 = 11): `void`, `()` 3, `{}`, `::`, `,`, `-`, `.`, the one + // written `;`, and `typename` (#1557). Operands (n2 = 7, N2 = 7): + // `f`, `MiddlePointer`, `quant_`, `order`, `2`, `Write`, `w`. + // Before #1546 the MISSING `;` was a second `;`. let cast = "void f() {\n typename Quant::MiddlePointer(quant_, order - 2).Write(w);\n}\n"; - assert_halstead_counts::(cast, "foo.cpp", [8, 10, 7, 7], label); + assert_halstead_counts::(cast, "foo.cpp", [9, 11, 7, 7], label); } /// Regression for #1546 on the two C++ clones. Mozcpp owns no file diff --git a/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap b/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap index 833713df..ca5892f2 100644 --- a/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap +++ b/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap @@ -3,8 +3,8 @@ source: tests/output_formats/csv_test.rs expression: out --- path,space_name,space_kind,start_line,end_line,cognitive.sum,cognitive.average,cognitive.min,cognitive.max,cyclomatic.sum,cyclomatic.average,cyclomatic.min,cyclomatic.max,cyclomatic.modified.sum,cyclomatic.modified.average,cyclomatic.modified.min,cyclomatic.modified.max,halstead.unique_operators,halstead.total_operators,halstead.unique_operands,halstead.total_operands,halstead.length,halstead.estimated_program_length,halstead.purity_ratio,halstead.vocabulary,halstead.volume,halstead.difficulty,halstead.level,halstead.effort,halstead.time,halstead.bugs,loc.sloc,loc.ploc,loc.lloc,loc.cloc,loc.blank,loc.sloc_average,loc.ploc_average,loc.lloc_average,loc.cloc_average,loc.blank_average,loc.sloc_min,loc.sloc_max,loc.cloc_min,loc.cloc_max,loc.ploc_min,loc.ploc_max,loc.lloc_min,loc.lloc_max,loc.blank_min,loc.blank_max,nom.functions,nom.closures,nom.functions_average,nom.closures_average,nom.total,nom.average,nom.functions_min,nom.functions_max,nom.closures_min,nom.closures_max,nargs.function_args,nargs.closure_args,nargs.function_args_average,nargs.closure_args_average,nargs.total,nargs.average,nargs.function_args_min,nargs.function_args_max,nargs.closure_args_min,nargs.closure_args_max,nexits.sum,nexits.average,nexits.min,nexits.max,tokens.sum,tokens.average,tokens.min,tokens.max,abc.assignments,abc.branches,abc.conditions,abc.magnitude,abc.assignments_average,abc.branches_average,abc.conditions_average,abc.assignments_min,abc.assignments_max,abc.branches_min,abc.branches_max,abc.conditions_min,abc.conditions_max,wmc.class_wmc_sum,wmc.interface_wmc_sum,wmc.total,npm.class_npm_sum,npm.interface_npm_sum,npm.class_methods,npm.interface_methods,npm.class_coa,npm.interface_coa,npm.total,npm.total_methods,npm.coa,npa.class_npa_sum,npa.interface_npa_sum,npa.class_attributes,npa.interface_attributes,npa.class_cda,npa.interface_cda,npa.total,npa.total_attributes,npa.cda,mi.original,mi.sei,mi.visual_studio -widget.cc,widget.cc,unit,1,10,0,0,0,0,5,2.5,1,1,5,2.5,1,1,9,19,6,9,28,44.039100017307746,1.5728250006181337,15,109.39293667703852,6.75,0.14814814814814814,738.40232257001,41.02235125388945,0.027231659910827242,10,9,2,0,1,2,1.8,0.4,0,0.2,1,10,0,0,1,9,1,2,0,1,2,0,0.4,0,2,0.4,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,7.6,0,12,1,0,0,1,0.2,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,108.13440061021349,80.81321081486506,63.23649158492017 -widget.cc,ns,namespace,2,10,0,0,0,0,4,2,1,1,4,2,1,1,9,19,6,9,28,44.039100017307746,1.5728250006181337,15,109.39293667703852,6.75,0.14814814814814814,738.40232257001,41.02235125388945,0.027231659910827242,9,9,2,0,0,2.25,2.25,0.5,0,0,1,9,0,0,1,9,1,2,0,0,2,0,0.5,0,2,0.5,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,9.5,5,12,1,0,0,1,0.25,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,110.0712409638703,83.50566092867489,64.36914676249725 +widget.cc,widget.cc,unit,1,10,0,0,0,0,5,2.5,1,1,5,2.5,1,1,10,20,6,9,29,48.72905595320056,1.680312274248295,16,116,7.5,0.13333333333333333,870,48.333333333333336,0.030377934896425568,10,9,2,0,1,2,1.8,0.4,0,0.2,1,10,0,0,1,9,1,2,0,1,2,0,0.4,0,2,0.4,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,7.6,0,12,1,0,0,1,0.2,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,107.82945249974337,80.37326368816134,63.05815935657507 +widget.cc,ns,namespace,2,10,0,0,0,0,4,2,1,1,4,2,1,1,10,20,6,9,29,48.72905595320056,1.680312274248295,16,116,7.5,0.13333333333333333,870,48.333333333333336,0.030377934896425568,9,9,2,0,0,2.25,2.25,0.5,0,0,1,9,0,0,1,9,1,2,0,0,2,0,0.5,0,2,0.5,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,9.5,5,12,1,0,0,1,0.25,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,109.76629285340017,83.06571380197117,64.19081453415215 widget.cc,Widget,class,3,9,0,0,0,0,3,1.5,1,1,3,1.5,1,1,9,17,5,8,25,40.13896548741762,1.6055586194967049,14,95.1838730514401,7.2,0.1388888888888889,685.3238859703688,38.07354922057604,0.02591051177740593,7,7,2,0,0,2.3333333333333335,2.3333333333333335,0.6666666666666666,0,0,1,7,0,0,1,7,1,2,0,0,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,33,11,10,12,1,0,0,1,0.3333333333333333,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,115.09604084632235,90.65309494114345,67.3076262259195 widget.cc,value,function,5,5,0,0,0,0,1,1,1,1,1,1,1,1,5,5,2,2,7,13.60964047443681,1.944234353490973,7,19.651484454403228,2.5,0.4,49.12871113600807,2.7293728408893374,0.004471319041702271,1,1,1,0,0,1,1,1,0,0,1,1,0,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,10,10,10,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,155.2836050228807,148.4278547652622,90.80912574437467 widget.cc,set,function,6,6,0,0,0,0,1,1,1,1,1,1,1,1,6,6,3,4,10,20.264662506490403,2.0264662506490403,9,31.69925001442312,4,0.25,126.79700005769249,7.044277780982916,0.008413009854528818,1,1,1,0,0,1,1,1,0,0,1,1,0,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,12,12,12,12,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,152.79727628688778,144.8408406278255,89.35513233151332 diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index 44c17e00..58a78b26 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit 44c17e002c1850b0a4e6bac32b66a2b66113c417 +Subproject commit 58a78b269e18a6508890ae5609f0fbb44c719cd3 From 2d138c3da443ef3c70f168102c50fb1abb18dd91 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:12:41 -0700 Subject: [PATCH 33/48] fix(utils): feed guarded calls to test-lang gates A call made inside an inner `#[cfg(feature = ...)]` dropped the languages it named, so a test whose every parser was guarded derived no needs. The gate then took it for an always-compiled caller and cleared the needs of each helper it called. `assert_operator_rows` had three such callers, so its gate was checked against nothing and the #1556 version lacking `kotlin` printed OK over an E0425. The turbofish was never the gap: `::` already counted. A guarded call now hands its gate's languages to the callee, and only unguarded calls mark a callee always-compiled. The fixed gate found one more helper of the same shape, `braced_word_string_verdicts` in checker.rs, ungated while every call sits behind `tcl` / `irules`; it now carries their `any(...)`. Fixes #1562 --- big-code-analysis-ast/src/checker.rs | 1 + utils/check-test-lang-gates-test.py | 91 ++++++++++++++++++++++++++++ utils/check-test-lang-gates.py | 79 ++++++++++++++++++++++-- 3 files changed, 166 insertions(+), 5 deletions(-) diff --git a/big-code-analysis-ast/src/checker.rs b/big-code-analysis-ast/src/checker.rs index a17245ca..02e3abe9 100644 --- a/big-code-analysis-ast/src/checker.rs +++ b/big-code-analysis-ast/src/checker.rs @@ -3398,6 +3398,7 @@ mod tests { /// Also asserts the two spellings agree off a known chain and off a /// `Node::parent` climb: the walk now threads a chain, and the /// answer must not depend on which it gets. + #[cfg(any(feature = "irules", feature = "tcl"))] fn braced_word_string_verdicts( label: &str, code: &[u8], diff --git a/utils/check-test-lang-gates-test.py b/utils/check-test-lang-gates-test.py index 26443b92..1cceddfa 100755 --- a/utils/check-test-lang-gates-test.py +++ b/utils/check-test-lang-gates-test.py @@ -831,6 +831,97 @@ def test_three_hops_through_a_helper_that_names_nothing(self) -> None: """ self.assertEqual(_offending(source), [("shared", ["python"])]) + GUARDED_CALLS = """ +#[cfg(test)] +mod tests { + #[cfg(%s)] + fn shared() {} + + #[cfg(any(feature = "python", feature = "typescript"))] + #[test] + fn guarded_case() { + #[cfg(feature = "python")] + shared::(); + #[cfg(feature = "typescript")] + shared::(); + } + + #[cfg(feature = "rust")] + #[test] + fn rust_case() { + shared::(); + } +} +""" + + def test_a_caller_whose_every_call_is_guarded_still_feeds_the_helper( + self, + ) -> None: + """#1562, the #1556 shape: `assert_operator_rows` lacked `kotlin`. + + `guarded_case` names every parser behind an inner `#[cfg]`, so its + own needs are empty. Before #1562 that read as an always-compiled + caller, which cleared `shared`'s needs, so its gate was checked + against nothing and the first row printed OK over an `E0425` on + the `rust` leg. The second row is the guarded languages reaching + `shared` at all; the third, that the caller's own gate is not + widened by them. + """ + self.assertEqual( + _offending( + self.GUARDED_CALLS % 'any(feature = "python", feature = "typescript")' + ), + [("shared", ["rust"])], + ) + self.assertEqual( + _offending(self.GUARDED_CALLS % 'feature = "rust"'), + [("shared", ["python", "typescript"])], + ) + self.assertEqual( + _offending( + self.GUARDED_CALLS + % 'any(feature = "python", feature = "rust", feature = "typescript")' + ), + [], + ) + + def test_a_guarded_call_takes_only_its_innermost_gates_languages( + self, + ) -> None: + """The outer block names both parsers; each call is under one. + + `py_only` is correctly gated on `python`. Attributing its call to + the outer `any(…)` block as well would hand it `typescript` and + report it stranded on the `typescript` leg, where nothing calls it. + """ + source = """ +#[cfg(test)] +mod tests { + #[cfg(feature = "python")] + fn py_only() {} + + #[cfg(feature = "typescript")] + fn ts_only() {} + + #[cfg(any(feature = "python", feature = "typescript"))] + #[test] + fn nested_case() { + #[cfg(any(feature = "python", feature = "typescript"))] + { + #[cfg(feature = "python")] + py_only::(); + #[cfg(feature = "typescript")] + ts_only::(); + } + } +} +""" + items = _scan(source) + needs = gate.resolve_needs(items) + self.assertEqual(needs[_named(items, "py_only").index], {"python"}) + self.assertEqual(needs[_named(items, "ts_only").index], {"typescript"}) + self.assertEqual(gate.offenders(items, needs), []) + def test_a_cycle_terminates_and_reports_each_narrow_link(self) -> None: """Each half of the cycle is reached from the other's test. diff --git a/utils/check-test-lang-gates.py b/utils/check-test-lang-gates.py index 3f4d9006..48182cdc 100755 --- a/utils/check-test-lang-gates.py +++ b/utils/check-test-lang-gates.py @@ -31,7 +31,9 @@ concrete ``*Parser`` aliases and ``LANG::`` literals in its body, ignoring anything already behind a nested ``#[cfg(feature = …)]``, plus transitively the needed set of - same-module helper ``fn``s it calls. + same-module helper ``fn``s it calls. A call *inside* such a nested + gate hands the languages named under it to the callee instead, so a + test whose every parser is guarded still gates its helpers (#1562). 2. **Declared gate** -- the item's own ``cfg`` conjoined with every enclosing ``mod``'s. 3. **Verdict** -- for each needed feature, evaluate the declared gate @@ -815,6 +817,13 @@ class Item: hand_written: frozenset[str] #: Call-shaped references in the body, for resolving `fn`s. calls: frozenset[str] + #: The subset of `calls` made outside every inner `#[cfg(feature = …)]`, + #: so in each build the item itself is compiled into. + unguarded_calls: frozenset[str] + #: `(callee, languages)` for each call made inside an inner + #: `#[cfg(feature = …)]`, with the languages named under that same + #: gate. The callee is live wherever the guarded call is (#1562). + guarded_calls: tuple[tuple[str, frozenset[str]], ...] #: Every identifier in the body, for resolving `const` / `type`. references: frozenset[str] parent_index: int | None @@ -1061,6 +1070,9 @@ def scan_source( macro_end ].count("}") macro_body = "\n".join(lines[index : macro_end + 1]) + macro_calls = frozenset( + m.group(1) for m in CALL_RE.finditer(macro_body) + ) items.append( Item( path=path, @@ -1090,9 +1102,9 @@ def scan_source( direct=frozenset(), hardcoded=frozenset(), hand_written=_hand_written_features(raw, attr_line), - calls=frozenset( - m.group(1) for m in CALL_RE.finditer(macro_body) - ), + calls=macro_calls, + unguarded_calls=macro_calls, + guarded_calls=(), references=frozenset( m.group(1) for m in REFERENCE_RE.finditer(macro_body) ), @@ -1195,9 +1207,12 @@ def scan_source( # the test got no gate and panicked without the grammar — the # unsafe direction. guarded: dict[int, list[str]] = {} - for first, last, predicate in _inner_cfg_extents(lines, raw, index, body_end): + extents = _inner_cfg_extents(lines, raw, index, body_end) + for first, last, predicate in extents: for guarded_line in range(first, last + 1): guarded.setdefault(guarded_line, []).append(predicate) + # The languages each guarded line names, for `guarded_calls`. + skipped: dict[int, set[str]] = {} # A helper that maps one of its arguments onto a `LANG` is a # dispatcher, and its arms are options rather than requirements. # `tests/api/suppression_test.rs`'s `analyze_lang` picks the @@ -1247,11 +1262,43 @@ def scan_source( continue feature = table[hit.group(0)] if any(not gate_admits(g, feature) for g in inner_gates): + skipped.setdefault(line_number, set()).add(feature) continue direct.add(feature) if not hit.group(0).startswith("LANG::"): hardcoded.add(feature) body_offset += len(lines[line_number]) + 1 + # A call under `#[cfg(feature = "javascript")]` is made only in a + # build with that grammar, so it is not an unconditional use of + # its callee. It is still a use: before #1562 the languages it + # named were dropped as "already conditional" and nothing handed + # them to the callee, so a test whose every parser sat behind an + # inner gate read as naming nothing, was taken for always + # compiled, and cleared the needs of the helper it called. That + # helper then had no derived gate at all, and one missing a + # caller's language passed while failing to compile (E0425). + # + # Each call is attributed to its *innermost* gate only. An outer + # `#[cfg(any(…))]` block's languages include every nested arm's, + # so pairing a call with the outer extent too would hand `a` in + # `{ #[cfg(js)] a(); #[cfg(ts)] b(); }` the `ts` its own gate + # rightly lacks, and report it stranded on that leg. + guarded_calls: list[tuple[str, frozenset[str]]] = [] + for n in guarded: + first, last = min( + ((f, l) for f, l, _ in extents if f <= n <= l), + key=lambda span: span[1] - span[0], + ) + languages = frozenset().union( + *(skipped.get(k, ()) for k in range(first, last + 1)) + ) + if languages: + guarded_calls.extend( + (call.group(1), languages) for call in CALL_RE.finditer(lines[n]) + ) + unguarded_body = "\n".join( + "" if n in guarded else lines[n] for n in range(index, body_end + 1) + ) item = Item( path=path, @@ -1276,6 +1323,10 @@ def scan_source( hardcoded=frozenset(hardcoded), hand_written=_hand_written_features(raw, attr_line), calls=frozenset(m.group(1) for m in CALL_RE.finditer(body)), + unguarded_calls=frozenset( + m.group(1) for m in CALL_RE.finditer(unguarded_body) + ), + guarded_calls=tuple(guarded_calls), references=frozenset(m.group(1) for m in REFERENCE_RE.finditer(body)), parent_index=parent_index, index=len(items), @@ -1584,6 +1635,20 @@ def reachable(item: Item, generic_too: bool) -> set[str]: if not gate_excludes(predicate, feature) } + # A call behind an inner `#[cfg]` hands its callee the languages named + # under that gate, and nothing to the caller's own `all(...)`: the + # caller is compiled without them, the call is not (#1562). Seeded + # here so the fixpoint below carries it on to the callee's callees. + for item in items: + predicate = effective_predicate(items, item) + for name, languages in item.guarded_calls: + callee_index = _resolve_call(items, scopes, item, name) + if callee_index is None or items[callee_index].is_test_fn: + continue + needs[callee_index] |= { + f for f in languages if not gate_excludes(predicate, f) + } + # A referent with even one unconditional caller must stay # unconditional itself, whatever its other callers need. Without # this the reverse propagation below gates a helper on the union of @@ -1642,6 +1707,10 @@ def reachable(item: Item, generic_too: bool) -> set[str]: continue if not _uses(item, items[index], strict=True): continue + # A call behind an inner `#[cfg]` is made only where that + # gate holds, so it makes nothing always-compiled (#1562). + if items[index].kind == "fn" and name not in item.unguarded_calls: + continue unconditional.add(index) needs[index] = set() changed = True From a11561e0b97b7c5fffe7f59387fd9cd7bd16c07a Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:30:01 -0700 Subject: [PATCH 34/48] fix(getter/kotlin): bill modifier keywords Java, Groovy and C# bill their declaration modifiers as Halstead operators; Kotlin billed none. Each modifier keyword leaf is now an operator: `enum`, `data`, `sealed`, `annotation`, `inner`, `value`, `companion`, the four visibilities, `abstract`, `final`, `open`, `override`, `lateinit`, `const`, the function, parameter and platform modifiers, `reified` and the variance `out` (`in` already billed). The `modifiers` / `*_modifier` wrappers stay unlisted, so each counts once. Every keyword has one kind, so `kind_id` keys it without splitting one word across the two `n1` maps. `reified` is spelled only as the named `reification_modifier` kind, so `KotlinCode::is_primitive` keys it by text and `bca ops` lists the word written. The same words used as names (`val open = 1`) parse as identifiers and stay operands. The #1556 rows for `enum class`, `data class` and `sealed interface` now score the modifier, and the bracket parity row bills `override`. No Kotlin corpus is snapshotted, so no integration snapshot moves. Fixes #1558 --- big-code-analysis-ast/src/checker/kotlin.rs | 11 ++ big-code-analysis-ast/src/getter/kotlin.rs | 22 +++- big-code-analysis-book/src/metrics.md | 9 +- src/metrics/halstead.rs | 115 ++++++++++++++++++-- tests/parity/abc_halstead_bracket_parity.rs | 17 +-- 5 files changed, 151 insertions(+), 23 deletions(-) diff --git a/big-code-analysis-ast/src/checker/kotlin.rs b/big-code-analysis-ast/src/checker/kotlin.rs index 93f18636..ebab1f32 100644 --- a/big-code-analysis-ast/src/checker/kotlin.rs +++ b/big-code-analysis-ast/src/checker/kotlin.rs @@ -75,4 +75,15 @@ impl Checker for KotlinCode { // followed by an `if_expression` (not an `if_statement`, and no // wrapping clause node). impl_is_else_if_prev_sibling!(Kotlin, IfExpression, Else); + + // `reified` is the one modifier the grammar spells as an aliased + // named kind, `reification_modifier`, rather than a keyword token. + // Keying it by `kind_id` would make `bca ops` list an operator + // named `reification_modifier`, which appears nowhere in the + // source; keying it by text lists `reified` (#1558). It has no + // other kind, so nothing splits across the two `n1` maps. + #[inline] + fn is_primitive(node: &Node) -> bool { + node.kind_id() == Kotlin::ReificationModifier + } } diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index e0ba1dc1..b57d0647 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -219,12 +219,26 @@ impl Getter for KotlinCode { | ReturnAT // Operator: other keywords. `interface` is the declaration // keyword leaf beside `class` and `object` (#1556); its - // `class_declaration` wrapper stays unlisted. `enum` in - // `enum class` is not here: the grammar makes it a - // `class_modifier`, the same node as `data`, `sealed` and - // `annotation`, and Kotlin bills no modifier. + // `class_declaration` wrapper stays unlisted. | Class | Interface | Fun | Object | Val | Var | In | Is | As | AsQMARK | BANGis | BANGin | Constructor + // Operator: modifier keywords, as Java, Groovy and C# bill + // theirs (#1558). Each is the keyword leaf; its + // `*_modifier` / `modifiers` wrappers stay unlisted, so one + // modifier bills once (grammar-dispatch section 5). Every + // keyword has exactly one kind, so `kind_id` keys it without + // splitting it across the two `n1` maps. `Annotation2` is the + // `annotation` of `annotation class`; `Annotation` is an `@N` + // use, which is no keyword. `in` already bills as a variance + // modifier through `In` above, so `out` joins it. `reified` + // has only the `ReificationModifier` kind, which + // `KotlinCode::is_primitive` keys by its text. + | Enum | Sealed | Annotation2 | Data | Inner | Value | Companion + | Public | Private | Protected | Internal + | Abstract | Final | Open | Override | Lateinit | Const + | Tailrec | Kotlin::Operator | Infix | Inline | External | Suspend + | Vararg | Noinline | Crossinline | ReificationModifier | Out + | Expect | Actual // Operator: brackets, separators, terminators | SEMI | COMMA | COLONCOLON | DOT | LBRACE | LBRACK | LPAREN // Operator: assignment and arithmetic diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 0aba4b98..147b6e7a 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -716,9 +716,12 @@ tokens involved the same way they spell real operators: their type-declaration keywords: `class`, `struct`, `union`, `enum`, `interface`, `record`, `trait`, `@interface`, `@implementation` and `@protocol`. Java's `C.class` is not billed: it stays one operand. - Kotlin's `interface` followed in #1556; the `enum` of `enum class` is - a modifier there, like `data` and `sealed`, and Kotlin bills no - modifier. #1557 added C++'s `namespace`, `template`, `typename`, + Kotlin's `interface` followed in #1556, and since #1558 Kotlin bills + its modifiers as Java, Groovy and C# bill theirs: the `enum` of + `enum class`, `data`, `sealed`, the visibility keywords, `open`, + `override`, `suspend`, `reified`, `out` and the rest each count once, + as the keyword and never as its `modifiers` wrapper. The same words + used as names (`val open = 1`) stay operands. #1557 added C++'s `namespace`, `template`, `typename`, `typedef` and `using` (C and Objective-C bill `typedef` too), so `template ` and `template ` each bill their keyword and score alike. It also added TypeScript's `type`, diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 0323c0aa..0c42612c 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -6261,9 +6261,9 @@ end", } /// #1556: Kotlin's `interface` bills as its `class` and `object` - /// do. The modifier rows pin the other half of the decision: `enum` - /// in `enum class` is a `class_modifier` like `data`, and neither - /// bills, so the three modifier rows score exactly as `class` does. + /// do. The modifier rows score one above the bare keyword since + /// #1558: `enum` in `enum class` is a `class_modifier` like `data` + /// and `sealed`, and Kotlin bills its modifiers as Java does. #[cfg(feature = "kotlin")] #[test] fn kotlin_declaration_keywords_are_operators_1556() { @@ -6274,12 +6274,12 @@ end", ("interface", "interface I {}\n", [2, 2]), // `object`, `{}` ("object", "object O {}\n", [2, 2]), - // `class`, `{}` — no `enum` - ("enum class", "enum class E { A }\n", [2, 2]), - // `class`, `{}` — no `data` - ("data class", "data class D {}\n", [2, 2]), - // `interface`, `{}` — no `sealed` - ("sealed interface", "sealed interface S {}\n", [2, 2]), + // `enum`, `class`, `{}` + ("enum class", "enum class E { A }\n", [3, 3]), + // `data`, `class`, `{}` + ("data class", "data class D {}\n", [3, 3]), + // `sealed`, `interface`, `{}` + ("sealed interface", "sealed interface S {}\n", [3, 3]), // `interface`, `class`, `:`, `{}` twice ( "implementing class", @@ -6290,6 +6290,103 @@ end", assert_operator_rows::("foo.kt", ROWS); } + /// #1558: Kotlin bills its modifier keywords as Java, Groovy and C# + /// bill theirs, one family per row, each keyword the leaf and never + /// its `*_modifier` wrapper. The last row is the control: the same + /// words used as names are operands, and add no operator. + #[cfg(feature = "kotlin")] + #[test] + fn kotlin_modifiers_are_operators_1558() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `public`, `private`, `protected`, `internal`, `val` x4, + // `=` x4, `class`, `{}` + ( + "visibility", + "class C {\n public val a = 1\n private val b = 1\n \ + protected val c = 1\n internal val d = 1\n}\n", + [8, 14], + ), + // `abstract` x2, `open`, `final`, `override`, `class`, `{}` x2, + // `fun` x3, `()` x3, `=` + ( + "inheritance and member", + "abstract class A {\n abstract fun f()\n open fun g() {}\n \ + final override fun toString() = \"\"\n}\n", + [9, 15], + ), + // `inner`, `value`, `annotation`, `class` x3, `()`, `val`, `:` + ( + "class", + "inner class I\nvalue class V(val x: Int)\nannotation class N\n", + [7, 9], + ), + // `lateinit`, `companion`, `const`, `class`, `object`, `var`, + // `val`, `:`, `=`, `{}` x2 + ( + "property and companion", + "class C {\n lateinit var a: String\n companion object {\n \ + const val b = 1\n }\n}\n", + [10, 11], + ), + // six modifiers, `fun` x6, `()` x6, `{}` x5 + ( + "function", + "tailrec fun a() {}\noperator fun b() {}\ninfix fun c() {}\n\ + inline fun d() {}\nexternal fun e()\nsuspend fun f() {}\n", + [9, 23], + ), + // `inline`, `noinline`, `crossinline`, `vararg`, `fun`, + // `()` x3, `:` x3, `->` x2, `,` x2, `{}` + ( + "parameter", + "inline fun f(noinline a: () -> Unit, crossinline b: () -> Unit, \ + vararg c: Int) {}\n", + [10, 16], + ), + // `inline`, `reified`, `in`, `out`, `fun`, `class`, `<` x2, + // `>` x2, `,`, `()`, `{}` + ( + "type parameter", + "inline fun f() {}\nclass Box\n", + [11, 13], + ), + // `expect`, `actual`, `fun` x2, `()` x2, `{}` + ("platform", "expect fun f()\nactual fun g() {}\n", [5, 7]), + // `fun`, `()`, `{}`: an `@N` use is the `annotation` node, not + // the keyword, so it bills no `annotation` operator + ("annotation use", "@N fun k() {}\n", [3, 3]), + // `val` x3, `=` x3: `open`, `out` and `value` are names here + ( + "names", + "val open = 1\nval out = 2\nval value = 3\n", + [2, 6], + ), + ]; + assert_operator_rows::("foo.kt", ROWS); + } + + /// #1558: `reified` is the one Kotlin modifier spelled as an aliased + /// named kind. `KotlinCode::is_primitive` keys it by its text, so + /// `bca ops` lists the word written, not the grammar's + /// `reification_modifier`. `n1` cannot see the difference — the kind + /// has one spelling either way — so the vocabulary is the only place + /// this is observable. + #[cfg(feature = "kotlin")] + #[test] + fn kotlin_reified_is_listed_by_its_text_1558() { + let ops = ops_of::("fun f() {}\n", "foo.kt"); + assert!( + ops.operators.iter().any(|op| op == "reified"), + "{:?}", + ops.operators + ); + assert!( + !ops.operators.iter().any(|op| op == "reification_modifier"), + "{:?}", + ops.operators + ); + } + /// #1557 rows shared by the two C++ clones: the declaration keywords /// #1552 left out, one per row, plus the two disambiguator positions /// (`typename T::x`, `x.template f`) that bill the same leaf. diff --git a/tests/parity/abc_halstead_bracket_parity.rs b/tests/parity/abc_halstead_bracket_parity.rs index 05e20477..14004c06 100644 --- a/tests/parity/abc_halstead_bracket_parity.rs +++ b/tests/parity/abc_halstead_bracket_parity.rs @@ -146,17 +146,20 @@ fn fixture(lang: LANG) -> Option<(&'static str, &'static str, &'static [&'static 22, ), // A qualified super call disambiguates its supertype with the - // same two bare tokens. `override` is in no arm of the Kotlin - // getter, which is why it is absent below. + // same two bare tokens. `override` bills as Kotlin's other + // modifiers do since #1558. // - // expected N1 = 13: the header's `class`, `:`, `()` and `{}`, - // then `fun`, the parameter `()`, the return-type `:`, the body - // `{}`, `return`, `<`, `>`, `.` and the call's `()`. + // expected N1 = 14: the header's `class`, `:`, `()` and `{}`, + // then `override`, `fun`, the parameter `()`, the return-type + // `:`, the body `{}`, `return`, `<`, `>`, `.` and the call's `()`. LANG::Kotlin => ( "class B : A() {\n override fun g(): Int { return super.g() }\n}\n", "kt", - ["()", ".", ":", "<", ">", "class", "fun", "return", "{}"].as_slice(), - 13, + [ + "()", ".", ":", "<", ">", "class", "fun", "override", "return", "{}", + ] + .as_slice(), + 14, ), // A filehandle readline. `` is deliberately not the // fixture: the grammar lexes it as one `standard_input` token, From f43e5d97f5c2394608e04d8b12351709120e447e Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:40:29 -0700 Subject: [PATCH 35/48] fix(getter/java): bill type names as operands Java classified no `type_identifier`, so a type name used as a type contributed no Halstead operand: `class C implements I { Foo f; }` billed `C`, `I` and `f` but not `Foo`, and the `implements I` reference billed nothing. C#, Rust, the C family, Go, Groovy and TypeScript (#1557) bill type names as operands; Java now does too. The grammar has one `type_identifier` kind and no listed node wraps it, so each name bills once: a field or parameter type, an `implements` / `extends` entry, a type parameter, a generic argument, and each segment of `java.util.List` (the `.` already bills). Two positions stay unbilled: - a type inside a class literal, directly or under an array or scoped type (`Foo.class`, `Foo[].class`, `java.util.List.class`), since the literal is already one operand keyed by its whole text; - `var` in `var x = 1`, which the grammar spells as a type name. It names no type, and C# bills its `var` as nothing, so `get_op_type_with_code` keeps it unbilled. A variable named `var` is an `identifier` and stays an operand. `java_operators_and_operands` and `java_ops` move by the one `string` parameter type (n2 12 -> 13, N2 22 -> 23). No Java corpus is snapshotted, so no integration snapshot moves. Fixes #1560 --- big-code-analysis-ast/src/getter/java.rs | 46 +++++++++- big-code-analysis-ast/src/getter/kotlin.rs | 9 +- big-code-analysis-book/src/metrics.md | 10 +- src/metrics/halstead.rs | 101 +++++++++++++++++---- src/ops.rs | 2 + 5 files changed, 143 insertions(+), 25 deletions(-) diff --git a/big-code-analysis-ast/src/getter/java.rs b/big-code-analysis-ast/src/getter/java.rs index 1cd8bb5b..15170899 100644 --- a/big-code-analysis-ast/src/getter/java.rs +++ b/big-code-analysis-ast/src/getter/java.rs @@ -90,6 +90,21 @@ impl Getter for JavaCode { // leaf too would pay the same bytes twice (grammar-dispatch // section 5, the compound-leaf guard). Class if ancestors.parent_has_kind(node, ClassLiteral as u16) => TokenRole::Unknown, + // A type name is an operand (#1560, the operand arm below) + // except inside a class literal, whose whole text is already + // one operand — the `class` guard above, for the same + // reason. The type sits there directly (`Foo.class`) or + // under an array or scoped type (`Foo[].class`, + // `java.util.List.class`), so the climb skips those two. + TypeIdentifier + if ancestors + .iter(node) + .map(|(ancestor, _)| ancestor.kind_id()) + .find(|&id| id != ArrayType as u16 && id != ScopedTypeIdentifier as u16) + == Some(ClassLiteral as u16) => + { + TokenRole::Unknown + } // Operator: control flow | If | Else | Switch | Case | Try | Catch | Throw | Throws | Throws2 | For | While | Continue | Break | Do | Finally @@ -135,8 +150,16 @@ impl Getter for JavaCode { // `csharp.rs`, deliberately: a parameter name is an operand // everywhere here, while a *member* named by a keyword sits // with `operator +`. - Identifier | NullLiteral | ClassLiteral | True | False | StringLiteral - | CharacterLiteral | HexIntegerLiteral | OctalIntegerLiteral + // + // `TypeIdentifier` joined in #1560, as C#, Rust, the C family, + // Go, Groovy and TypeScript bill type names: a field or + // parameter type, an `implements` / `extends` entry, a type + // parameter, a generic argument, each segment of + // `java.util.List` (the `.` bills as itself). It is the one + // type-name kind and no listed node wraps it, so each name + // bills once. + Identifier | TypeIdentifier | NullLiteral | ClassLiteral | True | False + | StringLiteral | CharacterLiteral | HexIntegerLiteral | OctalIntegerLiteral | BinaryIntegerLiteral | DecimalIntegerLiteral | HexFloatingPointLiteral | DecimalFloatingPointLiteral | This => { TokenRole::Operand @@ -147,6 +170,25 @@ impl Getter for JavaCode { } } + // `var x = …` parses `var` as a `type_identifier` (it is a reserved + // type name, not a keyword token), so the #1560 operand arm would + // bill an operand spelled `var` that names no type. It stays + // unbilled, as C#'s `var` (an `implicit_type` wrapping its own + // keyword) is; a variable *named* `var` is an `identifier` and is + // unaffected. Only the bytes tell the two type names apart, so the + // byte-less `get_op_type` above still answers `Operand` for it; the + // walk calls this spelling (grammar-dispatch section 7). + fn get_op_type_with_code<'a>( + node: &Node<'a>, + code: &[u8], + ancestors: Ancestors<'a, '_>, + ) -> TokenRole { + if node.kind_id() == Java::TypeIdentifier && node_text(code, node) == Some("var") { + return TokenRole::Unknown; + } + Self::get_op_type(node, ancestors) + } + fn get_operator_id_as_str(id: u16) -> &'static str { let typ = id.into(); match typ { diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index b57d0647..ee3f8e6e 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -222,10 +222,11 @@ impl Getter for KotlinCode { // `class_declaration` wrapper stays unlisted. | Class | Interface | Fun | Object | Val | Var | In | Is | As | AsQMARK | BANGis | BANGin | Constructor - // Operator: modifier keywords, as Java, Groovy and C# bill - // theirs (#1558). Each is the keyword leaf; its - // `*_modifier` / `modifiers` wrappers stay unlisted, so one - // modifier bills once (grammar-dispatch section 5). Every + // Operator: modifier keywords, as C# bills all of its own + // (#1558; Java and Groovy bill only some, #1565). Each is + // the keyword leaf; its `*_modifier` / `modifiers` wrappers + // stay unlisted, so one modifier bills once + // (grammar-dispatch section 5). Every // keyword has exactly one kind, so `kind_id` keys it without // splitting it across the two `n1` maps. `Annotation2` is the // `annotation` of `annotation class`; `Annotation` is an `@N` diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 147b6e7a..9c1f4d1f 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -717,7 +717,8 @@ tokens involved the same way they spell real operators: `interface`, `record`, `trait`, `@interface`, `@implementation` and `@protocol`. Java's `C.class` is not billed: it stays one operand. Kotlin's `interface` followed in #1556, and since #1558 Kotlin bills - its modifiers as Java, Groovy and C# bill theirs: the `enum` of + every modifier, as C# does (Java and Groovy bill `abstract` and + `final` but no visibility keyword or `static`, #1565): the `enum` of `enum class`, `data`, `sealed`, the visibility keywords, `open`, `override`, `suspend`, `reified`, `out` and the rest each count once, as the keyword and never as its `modifiers` wrapper. The same words @@ -727,7 +728,12 @@ tokens involved the same way they spell real operators: keyword and score alike. It also added TypeScript's `type`, `namespace`, `module`, `declare` and `global`, and made every TypeScript type name an operand (`x: Foo`, `interface I`, - `Map`), as C#, Rust and the C family already did. + `Map`), as C#, Rust and the C family already did. Java followed + in #1560: `Foo f`, `implements I`, `List` and each segment of + `java.util.List` bill their names as operands. A type inside a class + literal (`Foo.class`) does not, since the literal is already one + operand, and neither does the inferred type `var`, which C# bills + nothing for either. - **An alternative spelling is an operator of its own.** C++ spells `&&` equally as `and`, `!=` as `not_eq` and so on through all eleven [alternative tokens](https://en.cppreference.com/w/cpp/language/operator_alternative). diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 0c42612c..60670269 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -2672,25 +2672,26 @@ mod tests { "foo.java", |metric| { // Operators (n1=12): class {} void () [] , . ; int = + / - // Operands (n2=12): Main main args a b c avg 5 3 MessageFormat format "{0}" + // Operands (n2=13): Main main string args a b c avg 5 3 MessageFormat format "{0}" + // (`string` is the parameter's type name, an operand since #1560) insta::assert_json_snapshot!( metric.halstead, @r#" { "unique_operators": 12, "total_operators": 27, - "unique_operands": 12, - "total_operands": 22, - "length": 49, - "estimated_program_length": 86.03910001730775, - "purity_ratio": 1.7559000003532192, - "vocabulary": 24, - "volume": 224.66316253533665, - "difficulty": 11.0, - "level": 0.09090909090909091, - "effort": 2471.294787888703, - "time": 137.29415488270573, - "bugs": 0.060929616893854746 + "unique_operands": 13, + "total_operands": 23, + "length": 50, + "estimated_program_length": 91.12526634448807, + "purity_ratio": 1.8225053268897613, + "vocabulary": 25, + "volume": 232.19280948873623, + "difficulty": 10.615384615384615, + "level": 0.09420289855072464, + "effort": 2464.8159776496614, + "time": 136.93422098053674, + "bugs": 0.06082308053869885 } "# ); @@ -5928,7 +5929,7 @@ end", } /// Asserts `[n2, N2]` for each row. - #[cfg(feature = "typescript")] + #[cfg(any(feature = "java", feature = "typescript"))] fn assert_operand_rows(file: &str, rows: &[(&str, &str, [u64; 2])]) { assert_halstead_rows::(file, rows, |h| [h.unique_operands(), h.total_operands()]); } @@ -6140,6 +6141,72 @@ end", assert_operator_rows::("Foo.java", ROWS); } + /// #1560: a Java type name is an operand, as in C#, Rust, the C + /// family, Go, Groovy and TypeScript. Each row isolates one position + /// of the single `type_identifier` kind. The class-literal rows pin + /// that a type inside `Foo.class` is not billed again on top of the + /// literal's own operand, whether it sits there directly or under an + /// array or scoped type; the `var` rows, that the inferred-type + /// keyword bills nothing while a variable named `var` stays an + /// operand. + #[cfg(feature = "java")] + #[test] + fn java_type_names_are_operands_1560() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `C`, `I` x2, `Foo`, `f` + ( + "field type and implements", + "class C implements I { Foo f; }\ninterface I {}\n", + [4, 5], + ), + // `C`, `java`, `util`, `List`, `Foo`, `l` + ( + "scoped and generic", + "class C { java.util.List l; }\n", + [6, 6], + ), + // `C`, `T` x2, `Comparable` + ( + "type parameter", + "class C> {}\n", + [3, 4], + ), + // `C`, `m`, `Ex` + ("throws", "class C { void m() throws Ex {} }\n", [3, 3]), + // `C`, `Object`, `a`, `Foo.class` + ( + "class literal", + "class C { Object a = Foo.class; }\n", + [4, 4], + ), + // `C`, `Object`, `a`, `Foo[].class` + ( + "array class literal", + "class C { Object a = Foo[].class; }\n", + [4, 4], + ), + // `C`, `Object`, `a`, `java.util.List.class` + ( + "scoped class literal", + "class C { Object a = java.util.List.class; }\n", + [4, 4], + ), + // `C`, `m`, `x`, `1` — no `var` + ( + "inferred type", + "class C { void m() { var x = 1; } }\n", + [4, 4], + ), + // `C`, `m`, `var`, `1` — the variable's name + ( + "variable named var", + "class C { void m() { int var = 1; } }\n", + [4, 4], + ), + ]; + assert_operand_rows::("Foo.java", ROWS); + } + /// #1552: Groovy's type-declaration keywords are operators, as /// Java's are. `String.class` is a `field_access` whose `class` is a /// plain identifier, so it stays an operand with no guard needed. @@ -6263,7 +6330,7 @@ end", /// #1556: Kotlin's `interface` bills as its `class` and `object` /// do. The modifier rows score one above the bare keyword since /// #1558: `enum` in `enum class` is a `class_modifier` like `data` - /// and `sealed`, and Kotlin bills its modifiers as Java does. + /// and `sealed`, and Kotlin bills its modifiers as C# does. #[cfg(feature = "kotlin")] #[test] fn kotlin_declaration_keywords_are_operators_1556() { @@ -6290,8 +6357,8 @@ end", assert_operator_rows::("foo.kt", ROWS); } - /// #1558: Kotlin bills its modifier keywords as Java, Groovy and C# - /// bill theirs, one family per row, each keyword the leaf and never + /// #1558: Kotlin bills its modifier keywords as C# bills its own, + /// one family per row, each keyword the leaf and never /// its `*_modifier` wrapper. The last row is the control: the same /// words used as names are operands, and add no operator. #[cfg(feature = "kotlin")] diff --git a/src/ops.rs b/src/ops.rs index aafe2ef5..62c9468f 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -894,6 +894,8 @@ mod tests { &mut [ "Main", "main", + // the parameter's type name (#1560) + "string", "args", "a", "b", From 433b9edf12f78fb0fffc37ff79d7fd36e50c7a44 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:53:09 -0700 Subject: [PATCH 36/48] fix(getter/ts): bill type-level operator keywords TypeScript billed `as`, `typeof` and `extends` but not their type-level siblings. `satisfies`, `keyof`, `infer`, and the `is` and `asserts` of a type predicate are now operators in both TypeScript grammars, each as its keyword leaf. The `satisfies_expression`, `index_type_query`, `infer_type`, `type_predicate` and `asserts` wrappers stay unlisted, so a keyword bills once. `Asserts2` is the keyword token under the `Asserts` node; billing either is indistinguishable, and billing both is what the new rows catch. Used as names (`const keyof = 1`, `o.is`, a `satisfies` parameter) the words parse as identifiers and stay operands. The pinned grammar misreads one valid spelling, `let satisfies = 1`, as a `satisfies_expression` around an ERROR; that statement was already mis-scored and is noted in the getter rather than pinned. The keywords live in the TS/TSX extras only, so JavaScript, Mozjs and the pdf.js corpus do not move, and no snapshot moves. Fixes #1561 --- big-code-analysis-ast/src/getter/tsx.rs | 5 +- .../src/getter/typescript.rs | 18 +++++++ big-code-analysis-book/src/metrics.md | 5 +- src/metrics/halstead.rs | 50 +++++++++++++++++++ 4 files changed, 76 insertions(+), 2 deletions(-) diff --git a/big-code-analysis-ast/src/getter/tsx.rs b/big-code-analysis-ast/src/getter/tsx.rs index d50967c3..a95069c6 100644 --- a/big-code-analysis-ast/src/getter/tsx.rs +++ b/big-code-analysis-ast/src/getter/tsx.rs @@ -81,11 +81,14 @@ impl Getter for TsxCode { // // `Type` / `Namespace` / `Module2` / `Declare` / `Global` and the // `TypeIdentifier` operand: see the TS invocation (#1557). + // + // `Satisfies` / `Keyof` / `Infer` / `Is` / `Asserts2`, the type-level + // operator keywords: see the TS invocation (#1561). impl_js_family_get_op_type!( Tsx, op_extras: [ QMARKDOT, PredefinedType, Interface, Enum, Type, Namespace, Module2, Declare, Global, - LTSLASH, SLASHGT, + Satisfies, Keyof, Infer, Is, Asserts2, LTSLASH, SLASHGT, ], operand_extras: [Identifier2, String2, TypeIdentifier], predefined_void: PredefinedType, diff --git a/big-code-analysis-ast/src/getter/typescript.rs b/big-code-analysis-ast/src/getter/typescript.rs index d56a0a7e..f6356cf8 100644 --- a/big-code-analysis-ast/src/getter/typescript.rs +++ b/big-code-analysis-ast/src/getter/typescript.rs @@ -96,10 +96,28 @@ impl Getter for TypescriptCode { // every one while C#, Rust and the C family bill theirs. No // classified node wraps it (`generic_type` and // `nested_type_identifier` are unlisted), so it bills once. + // + // #1561 adds the type-level operator keywords, the siblings of the + // `as` / `typeof` / `extends` the macro already bills: `satisfies`, + // `keyof`, `infer`, the type-predicate `is` and `asserts`. Each is + // its keyword leaf; the `satisfies_expression`, `index_type_query`, + // `infer_type`, `type_predicate` and `asserts` wrappers stay + // unlisted, so one keyword bills once. `Asserts2` is the keyword + // token and the unsuffixed `Asserts` the node wrapping it, so + // `asserts a is T` bills `asserts` and `is` once each. Which of the + // two bills is unobservable — every node holds exactly one token and + // both render `asserts` — so the leaf is kept, as everywhere else + // here; listing both is what the tests catch. Used as names + // (`const keyof = 1`, `o.is`, a `satisfies` parameter) the words + // parse as identifiers. One valid spelling the pinned grammar gets + // wrong is `let satisfies = 1`: it reads `let` as an identifier and + // recovers a `satisfies_expression` around an ERROR, so that + // statement is mis-scored whatever this list holds. impl_js_family_get_op_type!( Typescript, op_extras: [ QMARKDOT, PredefinedType, Interface, Enum, Type, Namespace, Module2, Declare, Global, + Satisfies, Keyof, Infer, Is, Asserts2, ], operand_extras: [TypeIdentifier], predefined_void: PredefinedType, diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 9c1f4d1f..a1709911 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -733,7 +733,10 @@ tokens involved the same way they spell real operators: `java.util.List` bill their names as operands. A type inside a class literal (`Foo.class`) does not, since the literal is already one operand, and neither does the inferred type `var`, which C# bills - nothing for either. + nothing for either. #1561 made TypeScript's type-level operator + keywords operators, as their value-level siblings `as`, `typeof` + and `extends` already were: `satisfies`, `keyof`, `infer`, and the + `is` and `asserts` of a type predicate. - **An alternative spelling is an operator of its own.** C++ spells `&&` equally as `and`, `!=` as `not_eq` and so on through all eleven [alternative tokens](https://en.cppreference.com/w/cpp/language/operator_alternative). diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 60670269..adc91648 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -6576,6 +6576,56 @@ end", assert_operand_rows::("foo.tsx", TS_TYPE_NAME_OPERAND_ROWS); } + /// #1561: TypeScript's type-level operator keywords bill as their + /// value-level siblings `as`, `typeof` and `extends` do, each as its + /// keyword leaf under an unbilled wrapper. The `asserts a is Baz` row + /// pins that the `asserts` node and its keyword token bill once + /// between them, and the last row that the same words used as names + /// are operands. Both TypeScript grammars carry the change. + #[cfg(feature = "typescript")] + #[test] + fn typescript_type_operator_keywords_1561() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `let`, `=`, `satisfies`, `;` + ("satisfies", "let s = w satisfies Sat;\n", [4, 4]), + // `type`, `=`, `keyof`, `;` + ("keyof", "type K = keyof T;\n", [4, 4]), + // `type`, `<` x2, `>` x2, `=`, `extends`, `infer`, `?`, `:`, `;` + ( + "infer", + "type I = X extends Array ? E : X;\n", + [9, 11], + ), + // `function`, `()`, `:` x2, `unknown`, `is`, `{}`, `return`, `;` + ( + "type predicate", + "function g(a: unknown): a is Baz { return true; }\n", + [8, 9], + ), + // `function`, `()`, `:` x2, `unknown`, `asserts`, `is`, `{}` + ( + "asserts predicate", + "function h(a: unknown): asserts a is Baz {}\n", + [7, 8], + ), + // `function`, `()`, `:` x2, `unknown`, `asserts`, `{}` + ( + "bare asserts", + "function k(a: unknown): asserts a {}\n", + [6, 7], + ), + // `const`, `=` and `;` five times each: the words are names + ( + "names", + "const keyof = 1;\nconst infer = 2;\nconst is = 3;\nconst asserts = 4;\n\ + const satisfies = 5;\n", + [3, 15], + ), + ]; + assert_operator_rows::("foo.ts", ROWS); + assert_operator_rows::("foo.tsx", ROWS); + } + #[cfg(feature = "bash")] #[test] fn bash_operators_and_operands() { From 5df69a7cea8bf974ed628974efb81e9f269ea4e3 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:29:21 -0700 Subject: [PATCH 37/48] fix(halstead): bill generic angle brackets as <> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A type-argument, type-parameter or template list billed its `<` and `>` as two operators under the comparison operators' kind ids, so `Vec` and `a < b` shared a vocabulary entry. Every other bracket pair bills once on its opener (#1395). The list's `<` now bills as one `<>` operator and its `>` as nothing, gated on the parent being the list node. The grammars give the opener the comparison's kind id, so a new `Getter::get_operator_spelling` hook keys it by the fixed text `<>` instead. Comparisons and shifts are unchanged. Covered: Rust (type arguments and parameters, turbofish, `for<'a>`, `use`), Java, Groovy, C#, TypeScript, TSX, Kotlin, C++ and Mozcpp. Go's generics use `[]` and already billed a pair. Rust's ``, C#'s `delegate*<…>` and Kotlin's `super` keep `<` and `>`. N1 falls wherever generics appear; n1 rises by one where a comparison survives beside the new `<>`. Halstead and MI move in 858 C/C++, 64 Rust and 4 C# corpus snapshots, and the self-scan baseline is refreshed. Fixes #1559 --- .bca-baseline.toml | 122 ++++++------ big-code-analysis-ast/src/getter.rs | 38 +++- big-code-analysis-ast/src/getter/cpp.rs | 23 +++ big-code-analysis-ast/src/getter/csharp.rs | 20 ++ .../src/getter/generic_angles.rs | 53 +++++ big-code-analysis-ast/src/getter/groovy.rs | 25 +++ big-code-analysis-ast/src/getter/java.rs | 17 ++ big-code-analysis-ast/src/getter/kotlin.rs | 21 +- big-code-analysis-ast/src/getter/mozcpp.rs | 21 ++ big-code-analysis-ast/src/getter/rust.rs | 26 +++ big-code-analysis-ast/src/getter/tsx.rs | 11 ++ .../src/getter/typescript.rs | 16 ++ big-code-analysis-book/src/metrics.md | 27 ++- src/metrics/halstead.rs | 111 ++++++----- src/ops.rs | 185 ++++++++++++++++++ tests/parity/abc_halstead_bracket_parity.rs | 14 +- tests/repositories/big-code-analysis-output | 2 +- 17 files changed, 605 insertions(+), 127 deletions(-) create mode 100644 big-code-analysis-ast/src/getter/generic_angles.rs diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 29dd27d3..262cad40 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -189,19 +189,19 @@ value = 6.0 path = "big-code-analysis-ast/src/ast.rs" qualified = "build" metric = "halstead.effort" -value = 95280.22256472526 +value = 92648.17221763342 [[entry]] path = "big-code-analysis-ast/src/c_macro.rs" qualified = "replace" metric = "halstead.effort" -value = 87076.53005268889 +value = 86055.10448022334 [[entry]] path = "big-code-analysis-ast/src/c_macro.rs" qualified = "step_normal" metric = "halstead.effort" -value = 70691.13308026636 +value = 70211.8711610781 [[entry]] path = "big-code-analysis-ast/src/c_macro.rs" @@ -221,53 +221,59 @@ qualified = "step_raw_string" metric = "nargs" value = 5.0 +[[entry]] +path = "big-code-analysis-ast/src/getter.rs" +qualified = "" +metric = "loc.sloc" +value = 1264.0 + [[entry]] path = "big-code-analysis-ast/src/getter.rs" qualified = "Getter::is_braced_literal_slot" metric = "halstead.effort" -value = 50566.900032069454 +value = 48009.70056636167 [[entry]] path = "big-code-analysis-ast/src/getter/bash.rs" qualified = "BashCode::get_op_type" metric = "halstead.effort" -value = 61150.53156222458 +value = 57299.9090179332 [[entry]] path = "big-code-analysis-ast/src/getter/elixir.rs" qualified = "ElixirCode::get_op_type" metric = "halstead.effort" -value = 81284.21083173114 +value = 76840.62078430264 [[entry]] path = "big-code-analysis-ast/src/getter/irules.rs" qualified = "IrulesCode::get_op_type" metric = "halstead.effort" -value = 54835.65543871385 +value = 51477.285144213616 [[entry]] path = "big-code-analysis-ast/src/getter/perl.rs" qualified = "PerlCode::get_op_type" metric = "halstead.effort" -value = 137407.31980126494 +value = 131000.62269483795 [[entry]] path = "big-code-analysis-ast/src/getter/php.rs" qualified = "PhpCode::get_op_type" metric = "halstead.effort" -value = 96592.02057608281 +value = 92711.94569253008 [[entry]] path = "big-code-analysis-ast/src/getter/python.rs" qualified = "PythonCode::get_op_type" metric = "halstead.effort" -value = 64665.475362488345 +value = 61900.01893104293 [[entry]] path = "big-code-analysis-ast/src/getter/ruby.rs" qualified = "RubyCode::get_op_type" metric = "halstead.effort" -value = 127317.70075099175 +value = 120642.89096069236 [[entry]] path = "big-code-analysis-ast/src/node.rs" @@ -279,7 +285,7 @@ value = 36.0 path = "big-code-analysis-ast/src/parser.rs" qualified = "Parser::filters" metric = "halstead.effort" -value = 92491.6833558545 +value = 87390.78947368421 [[entry]] path = "big-code-analysis-ast/src/preproc.rs" @@ -309,19 +315,19 @@ value = 7.0 path = "big-code-analysis-cli/src/baseline.rs" qualified = "Baseline::from_str" metric = "halstead.effort" -value = 62587.95570171439 +value = 61984.213363755734 [[entry]] path = "big-code-analysis-cli/src/baseline_diff.rs" qualified = "BaselineDiff::compute" metric = "halstead.effort" -value = 124425.17843774539 +value = 118241.28751025953 [[entry]] path = "big-code-analysis-cli/src/check_format.rs" qualified = "write_github_annotations" metric = "halstead.effort" -value = 51030.97459989012 +value = 48005.61993457766 [[entry]] path = "big-code-analysis-cli/src/commands.rs" @@ -333,19 +339,13 @@ value = 136001.1105471366 path = "big-code-analysis-cli/src/commands/analyze.rs" qualified = "run_command_metrics" metric = "halstead.effort" -value = 74146.94504520179 - -[[entry]] -path = "big-code-analysis-cli/src/commands/analyze.rs" -qualified = "run_command_ops" -metric = "halstead.effort" -value = 49966.059463762176 +value = 70105.44625734187 [[entry]] path = "big-code-analysis-cli/src/commands/analyze.rs" qualified = "write_aggregate" metric = "halstead.effort" -value = 57901.393538811906 +value = 54557.29125972433 [[entry]] path = "big-code-analysis-cli/src/commands/check.rs" @@ -369,7 +369,7 @@ value = 6.0 path = "big-code-analysis-cli/src/commands/check.rs" qualified = "run_check" metric = "halstead.effort" -value = 67343.38425539367 +value = 63799.464271979734 [[entry]] path = "big-code-analysis-cli/src/commands/check/effective_config.rs" @@ -417,13 +417,13 @@ value = 39.06404996924922 path = "big-code-analysis-cli/src/commands/report.rs" qualified = "run_command_report" metric = "halstead.effort" -value = 67793.18512453695 +value = 63991.55666020266 [[entry]] path = "big-code-analysis-cli/src/diff.rs" qualified = "materialize_tree" metric = "halstead.effort" -value = 63004.79514502588 +value = 60574.68986587111 [[entry]] path = "big-code-analysis-cli/src/diff.rs" @@ -515,12 +515,6 @@ qualified = "dispatch_strip_comments" metric = "nargs" value = 6.0 -[[entry]] -path = "big-code-analysis-cli/src/dispatch.rs" -qualified = "validate_and_resolve_file" -metric = "halstead.effort" -value = 48161.176757774374 - [[entry]] path = "big-code-analysis-cli/src/html_report.rs" qualified = "generate_html_report_with_vcs" @@ -573,7 +567,7 @@ value = 7.0 path = "big-code-analysis-cli/src/lib.rs" qualified = "legacy_hint" metric = "halstead.effort" -value = 62321.99298455441 +value = 59351.805921378844 [[entry]] path = "big-code-analysis-cli/src/manifest.rs" @@ -639,13 +633,13 @@ value = 7.0 path = "big-code-analysis-cli/src/threshold_suggestion.rs" qualified = "closest_names" metric = "halstead.effort" -value = 58505.309528447164 +value = 59984.93899693509 [[entry]] path = "big-code-analysis-cli/src/threshold_suggestion.rs" qualified = "edit_distance_with_cutoff" metric = "halstead.effort" -value = 70743.85391258422 +value = 70177.90308128356 [[entry]] path = "big-code-analysis-cli/src/thresholds.rs" @@ -849,7 +843,7 @@ value = 747.0 path = "big-code-analysis-py/src/vcs.rs" qualified = "options_from" metric = "halstead.effort" -value = 50629.47325240143 +value = 47661.70050580648 [[entry]] path = "big-code-analysis-py/src/vcs.rs" @@ -873,13 +867,13 @@ value = 5.0 path = "big-code-analysis-web/src/web/negotiate.rs" qualified = "negotiate" metric = "halstead.effort" -value = 49354.63340093155 +value = 48620.735506493904 [[entry]] path = "big-code-analysis-web/src/web/server.rs" qualified = "run_parse" metric = "halstead.effort" -value = 87922.2135069964 +value = 84151.62410156948 [[entry]] path = "big-code-analysis-web/src/web/server.rs" @@ -903,7 +897,7 @@ value = 5.0 path = "big-code-analysis-web/src/web/vcs.rs" qualified = "options_from" metric = "halstead.effort" -value = 55077.016098147826 +value = 51846.28098938873 [[entry]] path = "big-code-analysis-web/src/web/vcs.rs" @@ -917,12 +911,6 @@ qualified = "KotlinCode::compute" metric = "cyclomatic" value = 20.0 -[[entry]] -path = "src/metrics/abc/ruby.rs" -qualified = "RubyCode::compute" -metric = "halstead.effort" -value = 48099.42954412001 - [[entry]] path = "src/metrics/abc/rust.rs" qualified = "RustCode::compute" @@ -1065,13 +1053,13 @@ value = 5.0 path = "src/metrics/loc/perl.rs" qualified = "PerlCode::compute" metric = "halstead.effort" -value = 54607.47985931332 +value = 51954.26146005158 [[entry]] path = "src/metrics/npa/php.rs" qualified = "PhpCode::compute" metric = "halstead.effort" -value = 50326.64993163575 +value = 47869.643574826616 [[entry]] path = "src/metrics/npa/python.rs" @@ -1083,7 +1071,7 @@ value = 6.0 path = "src/ops.rs" qualified = "ops_inner" metric = "halstead.effort" -value = 73118.98435042621 +value = 70903.25755192844 [[entry]] path = "src/output/checkstyle.rs" @@ -1095,7 +1083,7 @@ value = 7.0 path = "src/output/code_climate.rs" qualified = "write_code_climate" metric = "halstead.effort" -value = 65397.75866742601 +value = 64499.43780660971 [[entry]] path = "src/output/dump.rs" @@ -1197,7 +1185,7 @@ value = 5.0 path = "src/spaces/compute.rs" qualified = "compute_per_node" metric = "halstead.effort" -value = 68533.65287917512 +value = 63645.7536117776 [[entry]] path = "src/spaces/compute.rs" @@ -1209,7 +1197,7 @@ value = 7.0 path = "src/spaces/compute.rs" qualified = "metrics_inner" metric = "halstead.effort" -value = 123785.03296627598 +value = 121169.85621346734 [[entry]] path = "src/spaces/compute.rs" @@ -1227,7 +1215,7 @@ value = 6.0 path = "src/vcs/bus_factor.rs" qualified = "compute" metric = "halstead.effort" -value = 51947.0591918485 +value = 48322.36668354654 [[entry]] path = "src/vcs/bus_factor.rs" @@ -1239,7 +1227,7 @@ value = 15.0 path = "src/vcs/bus_factor.rs" qualified = "greedy_truck_factor" metric = "halstead.effort" -value = 47960.96527182218 +value = 47756.87605789953 [[entry]] path = "src/vcs/bus_factor.rs" @@ -1275,7 +1263,7 @@ value = 6.0 path = "src/vcs/git/cached.rs" qualified = "build_cached" metric = "halstead.effort" -value = 78921.96719178827 +value = 78743.41070492903 [[entry]] path = "src/vcs/git/cached.rs" @@ -1305,13 +1293,13 @@ value = 6.0 path = "src/vcs/git/diff_parse.rs" qualified = "unquote_git_path" metric = "halstead.effort" -value = 74373.33475054632 +value = 74116.87497554444 [[entry]] path = "src/vcs/git/history.rs" qualified = "collect_events" metric = "halstead.effort" -value = 52128.25957334215 +value = 51486.680993977934 [[entry]] path = "src/vcs/git/history.rs" @@ -1349,12 +1337,6 @@ qualified = "process_commit" metric = "nexits" value = 9.0 -[[entry]] -path = "src/vcs/git/jit.rs" -qualified = "collect_touched" -metric = "halstead.effort" -value = 49957.782203054114 - [[entry]] path = "src/vcs/git/jit.rs" qualified = "collect_touched" @@ -1379,6 +1361,12 @@ qualified = "compute_features" metric = "nexits" value = 5.0 +[[entry]] +path = "src/vcs/git/jit.rs" +qualified = "experience_features" +metric = "halstead.effort" +value = 49040.88359210979 + [[entry]] path = "src/vcs/git/jit.rs" qualified = "experience_features" @@ -1395,7 +1383,7 @@ value = 6.0 path = "src/vcs/git/jit.rs" qualified = "history_features" metric = "halstead.effort" -value = 56372.217728048636 +value = 54100.245911755286 [[entry]] path = "src/vcs/git/jit.rs" @@ -1461,25 +1449,25 @@ value = 5.0 path = "src/vcs/replay.rs" qualified = "replay" metric = "halstead.effort" -value = 49393.80551407597 +value = 48730.801413215886 [[entry]] path = "src/vcs/score.rs" qualified = "apply_percentile" metric = "halstead.effort" -value = 71674.67451821786 +value = 71452.77150113358 [[entry]] path = "src/vcs/stats.rs" qualified = "Accumulator::finalize" metric = "halstead.effort" -value = 89416.15847479667 +value = 89220.9266877338 [[entry]] path = "src/vcs/stats.rs" qualified = "Accumulator::record" metric = "halstead.effort" -value = 59083.201813811575 +value = 58878.05180751363 [[entry]] path = "src/vcs/trend.rs" diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index b1d471ec..75d6d2dd 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -17,6 +17,7 @@ use crate::lang_helpers::tcl_family::{ }; use crate::space_kind::SpaceKind; use crate::traits::Search; +use generic_angles::GenericAngleKinds; use crate::*; @@ -175,11 +176,18 @@ macro_rules! impl_js_family_get_op_type { op_extras: [$($op_extra:ident),* $(,)?], operand_extras: [$($operand_extra:ident),* $(,)?] $(, predefined_void: $predefined_type:ident)? + $(, generic_angles: $generic_angles:ident)? $(, jsx: [$jsx_text:ident, $jsx_entity:ident])? $(,)? ) => { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use $lang::*; + $( + if $generic_angles.is_closer(node, ancestors) { + return TokenRole::Unknown; + } + )? + // TS/TSX only: a `void` return / parameter type is parsed as a // `predefined_type` wrapper around an inner `void` token. Both // the wrapper (routed through `is_primitive` into the text-keyed @@ -315,6 +323,15 @@ macro_rules! impl_js_family_get_op_type { } } + $( + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + $generic_angles.opener_spelling(node, ancestors) + } + )? + $( fn get_op_type_with_code<'a>( node: &Node<'a>, @@ -551,7 +568,10 @@ pub trait Getter { /// a script-taking command. /// /// A bracket delimiting *syntax* fits neither shape: no literal - /// would absorb its contribution, so it stays an operator. + /// would absorb its contribution, so it stays an operator. A + /// generic list's `<` / `>` is billed like the other bracket pairs, + /// once as `<>` on the opener ([`Self::get_operator_spelling`], #1559), + /// which leaves the comparison `<` and `>` their own entries. /// /// `ancestors` is the chain the walker descended through. Most /// impls read a parent from it to disambiguate a token whose role @@ -591,6 +611,21 @@ pub trait Getter { Self::get_op_type(node, ancestors) } + /// The fixed spelling an operator `node` is keyed under in place of + /// its kind id, for a token whose kind the grammar shares with a + /// different operator; `None` keys it by kind. A generic list's `<` + /// is the case (#1559): it has the comparison's kind but is keyed as + /// the `"<>"` pair, as `()`, `[]` and `{}` are. Consulted only for a + /// node [`get_op_type`](Self::get_op_type) calls an operator. + #[inline] + #[must_use] + fn get_operator_spelling<'a>( + _node: &Node<'a>, + _ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + None + } + /// Returns the source-byte slice used to key a Halstead *operand*. /// The default keys on the operand node's full byte range. Kotlin /// overrides this to narrow a short-interpolation name token @@ -1202,6 +1237,7 @@ mod ccomment; mod cpp; mod csharp; mod elixir; +mod generic_angles; mod go; mod groovy; mod irules; diff --git a/big-code-analysis-ast/src/getter/cpp.rs b/big-code-analysis-ast/src/getter/cpp.rs index b73b611a..b3d3f983 100644 --- a/big-code-analysis-ast/src/getter/cpp.rs +++ b/big-code-analysis-ast/src/getter/cpp.rs @@ -4,6 +4,18 @@ use super::*; use crate::c_declarator::declarator_name; +// `GT2` is the template closer's own symbol, which the parser folds +// onto `GT` (`cpp_template_closer_alias_never_reaches_kind_id`); it is +// listed defensively. +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Cpp::TemplateArgumentList as u16, + Cpp::TemplateParameterList as u16, + ], + openers: &[Cpp::LT as u16], + closers: &[Cpp::GT as u16, Cpp::GT2 as u16], +}; + impl Getter for CppCode { fn get_func_space_name<'a, 'tree>( node: &Node<'tree>, @@ -76,6 +88,10 @@ impl Getter for CppCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Cpp::*; + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } + // `LPAREN2` here (and the `LBRACK2`/`LBRACK3` aliases in the // Elixir/Ruby impls) is a defensive arm, not an active one: every // grammar's `public_symbol_map` collapses the second-alias opener @@ -262,5 +278,12 @@ impl Getter for CppCode { } } + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + get_operator!(Cpp); } diff --git a/big-code-analysis-ast/src/getter/csharp.rs b/big-code-analysis-ast/src/getter/csharp.rs index 74bc6c0f..84a24f9b 100644 --- a/big-code-analysis-ast/src/getter/csharp.rs +++ b/big-code-analysis-ast/src/getter/csharp.rs @@ -3,6 +3,15 @@ use super::*; +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Csharp::TypeArgumentList as u16, + Csharp::TypeParameterList as u16, + ], + openers: &[Csharp::LT as u16], + closers: &[Csharp::GT as u16], +}; + impl Getter for CsharpCode { fn get_space_kind(node: &Node) -> SpaceKind { use Csharp::*; @@ -48,6 +57,10 @@ impl Getter for CsharpCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Csharp::*; + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } + match node.kind_id().into() { // Control-flow keywords If | Else | Switch | Case | Default | Try | Catch | Finally | Throw @@ -227,5 +240,12 @@ impl Getter for CsharpCode { } } + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + get_operator!(Csharp); } diff --git a/big-code-analysis-ast/src/getter/generic_angles.rs b/big-code-analysis-ast/src/getter/generic_angles.rs new file mode 100644 index 00000000..cb0fc2c6 --- /dev/null +++ b/big-code-analysis-ast/src/getter/generic_angles.rs @@ -0,0 +1,53 @@ +//! The generic angle brackets every grammar with generics shares with +//! its comparison operators (#1559). + +use crate::{Ancestors, Node}; + +/// How [`get_operator_spelling`] renders a generic angle-bracket pair, +/// in every language that has one (#1559). +/// +/// [`get_operator_spelling`]: crate::Getter::get_operator_spelling +pub(crate) const ANGLE_PAIR: &str = "<>"; + +/// The kinds one grammar spells its generic angle brackets with: the +/// list nodes whose `<` / `>` delimit type arguments or parameters +/// rather than compare, and the bracket tokens themselves. +/// +/// A list's brackets are one bracket pair, billed like `()`, `[]` and +/// `{}` (#1395): the opener carries the pair as [`ANGLE_PAIR`] and the +/// closer bills nothing (#1559). The grammars give the brackets the +/// same kind ids as the comparison operators, so only the parent tells +/// the two apart, and the opener is keyed by text rather than by kind +/// to keep it out of the comparison `<`'s vocabulary entry. +pub(crate) struct GenericAngleKinds { + /// The list nodes: type-argument, type-parameter and template + /// lists, plus each grammar's other spelling of a generic list. + pub(crate) lists: &'static [u16], + /// The `<` kinds, with any alias the grammar declares for it. + pub(crate) openers: &'static [u16], + /// The `>` kinds, likewise. + pub(crate) closers: &'static [u16], +} + +impl GenericAngleKinds { + fn parent_is_list<'a>(&self, node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + ancestors + .parent(node) + .is_some_and(|parent| self.lists.contains(&parent.kind_id())) + } + + /// Whether `node` closes a generic list, and so bills nothing. + pub(crate) fn is_closer<'a>(&self, node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + self.closers.contains(&node.kind_id()) && self.parent_is_list(node, ancestors) + } + + /// [`ANGLE_PAIR`] when `node` opens a generic list, else `None`. + pub(crate) fn opener_spelling<'a>( + &self, + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + (self.openers.contains(&node.kind_id()) && self.parent_is_list(node, ancestors)) + .then_some(ANGLE_PAIR) + } +} diff --git a/big-code-analysis-ast/src/getter/groovy.rs b/big-code-analysis-ast/src/getter/groovy.rs index 5f5d5be2..a268f6c7 100644 --- a/big-code-analysis-ast/src/getter/groovy.rs +++ b/big-code-analysis-ast/src/getter/groovy.rs @@ -3,6 +3,20 @@ use super::*; +// `method_type_parameters` is the method-level spelling of +// `type_parameters` (` T id(T x)`). `LT2` is the `token.immediate` +// `<` that `type_parameters` and `type_arguments` open with; the parser +// reports it as `LT`, so it is listed defensively. +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Groovy::TypeArguments as u16, + Groovy::TypeParameters as u16, + Groovy::MethodTypeParameters as u16, + ], + openers: &[Groovy::LT as u16, Groovy::LT2 as u16], + closers: &[Groovy::GT as u16], +}; + impl Getter for GroovyCode { /// Names the space, synthesising one for constructs that carry no /// name token (#1184). @@ -82,6 +96,10 @@ impl Getter for GroovyCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Groovy::*; + + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } // Mirrors `JavaCode`'s minimal classification — modifiers // (`Public`, `Static`, …) and module keywords (`Package`, // `Import`, …) are excluded. The type-declaration keywords @@ -248,6 +266,13 @@ impl Getter for GroovyCode { } } + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + fn get_operator_id_as_str(id: u16) -> &'static str { let typ = id.into(); match typ { diff --git a/big-code-analysis-ast/src/getter/java.rs b/big-code-analysis-ast/src/getter/java.rs index 15170899..6dc818c7 100644 --- a/big-code-analysis-ast/src/getter/java.rs +++ b/big-code-analysis-ast/src/getter/java.rs @@ -3,6 +3,12 @@ use super::*; +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[Java::TypeArguments as u16, Java::TypeParameters as u16], + openers: &[Java::LT as u16], + closers: &[Java::GT as u16], +}; + impl Getter for JavaCode { /// Names the space, synthesising one for constructs that carry no /// name token (#1184). @@ -80,6 +86,10 @@ impl Getter for JavaCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Java::*; + + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } // Some guides that informed grammar choice for Halstead // keywords, operators, literals: https://docs.oracle.com/javase/specs/jls/se18/html/jls-3.html#jls-3.12 // https://www.geeksforgeeks.org/software-engineering-halsteads-software-metrics/?msclkid=5e181114abef11ecbb03527e95a34828 @@ -189,6 +199,13 @@ impl Getter for JavaCode { Self::get_op_type(node, ancestors) } + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + fn get_operator_id_as_str(id: u16) -> &'static str { let typ = id.into(); match typ { diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index ee3f8e6e..762130a8 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -122,6 +122,14 @@ fn kotlin_string_has_interp(node: &Node, code: &[u8]) -> bool { }) } +// `super` (a `super_expression`) names a supertype rather than +// listing type arguments, so its brackets stay `<` and `>`. +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[Kotlin::TypeArguments as u16, Kotlin::TypeParameters as u16], + openers: &[Kotlin::LT as u16], + closers: &[Kotlin::GT as u16], +}; + impl Getter for KotlinCode { /// Names the space, synthesising one for constructs that carry no /// name token (#1184). @@ -210,9 +218,13 @@ impl Getter for KotlinCode { } } - fn get_op_type<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> TokenRole { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Kotlin::*; + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } + match node.kind_id().into() { // Operator: control flow keywords If | Else | When | For | While | Do | Try | Catch | Finally | Throw | Return @@ -341,5 +353,12 @@ impl Getter for KotlinCode { } } + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + get_operator!(Kotlin); } diff --git a/big-code-analysis-ast/src/getter/mozcpp.rs b/big-code-analysis-ast/src/getter/mozcpp.rs index eab281af..e2d8435d 100644 --- a/big-code-analysis-ast/src/getter/mozcpp.rs +++ b/big-code-analysis-ast/src/getter/mozcpp.rs @@ -4,6 +4,16 @@ use super::*; use crate::c_declarator::declarator_name; +// `GT2` is listed defensively, as in the Cpp getter. +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Mozcpp::TemplateArgumentList as u16, + Mozcpp::TemplateParameterList as u16, + ], + openers: &[Mozcpp::LT as u16], + closers: &[Mozcpp::GT as u16, Mozcpp::GT2 as u16], +}; + impl Getter for MozcppCode { fn get_func_space_name<'a, 'tree>( node: &Node<'tree>, @@ -76,6 +86,10 @@ impl Getter for MozcppCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Mozcpp::*; + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } + // `LPAREN2` is a defensive arm (collapsed to `LPAREN` before // `kind_id()`; #768, see the Cpp note). match node.kind_id().into() { @@ -154,5 +168,12 @@ impl Getter for MozcppCode { } } + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + get_operator!(Mozcpp); } diff --git a/big-code-analysis-ast/src/getter/rust.rs b/big-code-analysis-ast/src/getter/rust.rs index 7eea8c29..8a0bb097 100644 --- a/big-code-analysis-ast/src/getter/rust.rs +++ b/big-code-analysis-ast/src/getter/rust.rs @@ -3,6 +3,22 @@ use super::*; +// `for_lifetimes` (`for<'a> fn(&'a u8)`) and `use_bounds` +// (`impl Tr + use<'a, T>`) are generic-parameter lists under other +// names. `LT2` is the `token(prec(1, '<'))` spelling `type_arguments` +// and `use_bounds` open with; the parser reports it as `LT`, so it is +// listed defensively. +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Rust::TypeArguments as u16, + Rust::TypeParameters as u16, + Rust::ForLifetimes as u16, + Rust::UseBounds as u16, + ], + openers: &[Rust::LT as u16, Rust::LT2 as u16], + closers: &[Rust::GT as u16], +}; + impl Getter for RustCode { fn get_func_space_name<'a, 'tree>( node: &Node<'tree>, @@ -36,6 +52,9 @@ impl Getter for RustCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Rust::*; + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } match node.kind_id().into() { // `||` is treated as an operator only if it's part of a binary expression. // This prevents misclassification inside macros where closures without arguments (e.g., `let closure = || { /* ... */ };`) @@ -93,5 +112,12 @@ impl Getter for RustCode { } } + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + get_operator!(Rust); } diff --git a/big-code-analysis-ast/src/getter/tsx.rs b/big-code-analysis-ast/src/getter/tsx.rs index a95069c6..7db304df 100644 --- a/big-code-analysis-ast/src/getter/tsx.rs +++ b/big-code-analysis-ast/src/getter/tsx.rs @@ -3,6 +3,12 @@ use super::*; +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[Tsx::TypeArguments as u16, Tsx::TypeParameters as u16], + openers: &[Tsx::LT as u16], + closers: &[Tsx::GT as u16], +}; + impl Getter for TsxCode { fn get_space_kind(node: &Node) -> SpaceKind { use Tsx::*; @@ -84,6 +90,10 @@ impl Getter for TsxCode { // // `Satisfies` / `Keyof` / `Infer` / `Is` / `Asserts2`, the type-level // operator keywords: see the TS invocation (#1561). + // + // `generic_angles`: see the TS invocation (#1559). A JSX tag's `<` / + // `>` hang from the JSX element nodes, not from a generic list, so + // they stay `<` and `>`. impl_js_family_get_op_type!( Tsx, op_extras: [ @@ -92,6 +102,7 @@ impl Getter for TsxCode { ], operand_extras: [Identifier2, String2, TypeIdentifier], predefined_void: PredefinedType, + generic_angles: GENERIC_ANGLES, jsx: [JsxText, HtmlCharacterReference], ); diff --git a/big-code-analysis-ast/src/getter/typescript.rs b/big-code-analysis-ast/src/getter/typescript.rs index f6356cf8..3565d8b6 100644 --- a/big-code-analysis-ast/src/getter/typescript.rs +++ b/big-code-analysis-ast/src/getter/typescript.rs @@ -3,6 +3,15 @@ use super::*; +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Typescript::TypeArguments as u16, + Typescript::TypeParameters as u16, + ], + openers: &[Typescript::LT as u16], + closers: &[Typescript::GT as u16], +}; + impl Getter for TypescriptCode { fn get_space_kind(node: &Node) -> SpaceKind { use Typescript::*; @@ -113,6 +122,12 @@ impl Getter for TypescriptCode { // wrong is `let satisfies = 1`: it reads `let` as an identifier and // recovers a `satisfies_expression` around an ERROR, so that // statement is mis-scored whatever this list holds. + // + // `generic_angles`: a type-argument or type-parameter list's `<` + // bills as the `<>` pair and its `>` as nothing (#1559). A `x` + // type assertion wraps a `type_arguments` node, so it is a pair too; + // a comparison's `<` / `>` sit under `binary_expression` and stay + // their own operators. impl_js_family_get_op_type!( Typescript, op_extras: [ @@ -121,6 +136,7 @@ impl Getter for TypescriptCode { ], operand_extras: [TypeIdentifier], predefined_void: PredefinedType, + generic_angles: GENERIC_ANGLES, ); get_operator!(Typescript); diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index a1709911..796bb44f 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -159,7 +159,7 @@ application would over-count. | Ruby | A subject-less `case` scores each `when` pattern as an `\|\|` operand; a subject-ful `case` scores per clause | A subject-less clause (`case; when x > 5`) lists ordinary boolean expressions, evaluated like an `if` predicate, and several patterns (`when a, b`) are an implicit `\|\|`. Each pattern therefore scores once unless its own operator already counted it, so `when x > 5` scores 1 like `if x > 5`, and `when a, b` scores 2 like `if a \|\| b`. Before this, a blanket per-clause increment counted a comparison twice. A subject-ful clause (`case x; when 1`) matches each pattern with an implicit `pattern === x` that nothing in the source spells, so the clause pays once (#1453). | | Go | A tagless `switch` scores each case expression as an `\|\|` operand; a tagged `switch` scores per case | A tagless switch (`switch { case x > 5: }`, with or without an init statement) is `switch true`: each case expression is an ordinary boolean scored by the same slot an `if` predicate uses, and a list (`case a, b:`) is an implicit `\|\|`. So `case x > 5:` scores 1 like `if x > 5`, and `case a, b:` scores 2 like `if a \|\| b`. Before this, a blanket per-case increment counted a comparison twice. A tagged case (`switch x { case 1: }`) compares with an implicit `x == 1` that nothing in the source spells, so the case pays once; type-switch and `select` cases are unchanged (#1523). | | Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive` (its `after` timeout included), of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). | -| C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these `<` / `>` as an operator, and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | +| C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these brackets as an operator (a generic list's as one `<>` pair since #1559), and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | | C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | | Perl | A comparison's right operand is not a condition, although the grammar parses it as one | tree-sitter-perl 1.1.2 lets a chaining comparison (`<`, `>`, `<=`, `>=`, `lt`, `gt`, `le`, `ge`, `==`, `!=`, `eq`, `ne`) take everything of lower precedence to its right as its operand, so `$a > $b ? 1 : 2` parses as `$a > ($b ? 1 : 2)` and `$a > $b && $c` as `$a > ($b && $c)`. The ternary's condition slot and the chain's first operand then held `$b`, which scored as a condition of its own: one above the JavaScript twin for each misparsed comparison. The walker recognises the shape and leaves that operand to the comparison, so both spellings score what the parenthesised `($a > $b) ? 1 : 2` and `($a > $b) && $c` do. An explicit `$a > ($b ? 1 : 2)` is unaffected. `<=>`, `cmp`, `=~` and `!~` do not chain and parse correctly. The workaround goes when the grammar is fixed upstream (#1273). | @@ -616,7 +616,7 @@ Closing brackets, comments and a literal's own delimiters count as nothing, which is why the Halstead totals are *not* the same as the Tokens count. -Seven classification rules are worth knowing because they are choices +The classification rules below are worth knowing because they are choices rather than consequences, and because several grammars spell the tokens involved the same way they spell real operators: @@ -737,6 +737,29 @@ tokens involved the same way they spell real operators: keywords operators, as their value-level siblings `as`, `typeof` and `extends` already were: `satisfies`, `keyof`, `infer`, and the `is` and `asserts` of a type predicate. +- **A generic list's angle brackets are one bracket pair.** The `<` + and `>` around type arguments and type parameters count as one `<>` + operator, billed at the `<` as `()` is at the `(`, and the `>` counts + nothing (#1559). That covers `Vec`, `fn f`, a turbofish + `h::()`, `for<'a>` and `use` in Rust; C++'s + `template ` and `std::vector`; and the type-argument + and type-parameter lists of Java, Groovy, C#, Kotlin, TypeScript and + TSX. A comparison's `<` and `>` and a shift's `<<` and `>>` stay + separate operators, and a C++ `>>` that closes two template lists + parses as two `>`, so it bills nothing. Go's generics use square + brackets and already billed `[]`. Angle brackets around anything + other than a generic list still bill `<` and `>`: a Rust + `::` path, a C# `delegate*` pointer type and + a Kotlin `super`, as well as the JSX, Lua and Perl constructs + above. So does a Kotlin generic call with arguments, `id(a)`, + which tree-sitter-kotlin-ng parses as two comparisons (#1394). + Before #1559 each bracket billed under the comparison's own entry, + so `Vec` and `a < b` shared one. Every generic now costs + one operator occurrence less, so `N1` falls in any code that uses + generics. `n1` rises by one where a comparison `<` or `>` remains + beside the new `<>`, so `halstead.effort` falls in most files and + rises in the rest; on the C++ and Rust corpora it fell in 87% of the + files that moved, and MI rose in most of them. - **An alternative spelling is an operator of its own.** C++ spells `&&` equally as `and`, `!=` as `not_eq` and so on through all eleven [alternative tokens](https://en.cppreference.com/w/cpp/language/operator_alternative). diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index adc91648..a9b4519b 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -75,7 +75,10 @@ pub struct HalsteadMaps<'a> { pub(crate) operators: IntKeyHashMap, /// Primitive-type operators stored by text so each distinct primitive /// (e.g. `int` vs `double`) counts as a separate distinct operator, - /// even when the grammar maps them all to a single kind_id. + /// even when the grammar maps them all to a single kind_id. Also + /// holds the operators a getter keys by a fixed spelling + /// ([`Getter::get_operator_spelling`]), such as the generic `<>` + /// pair (#1559). /// /// Text-keyed, so it keeps SipHash — see the module doc on /// [`crate::int_hash`] for why analysed source text does not qualify @@ -396,7 +399,12 @@ fn compute_halstead<'a, T: Getter + Checker>( } match T::get_op_type_with_code(node, code, ancestors) { TokenRole::Operator => { - if T::is_primitive(node) { + if let Some(spelling) = T::get_operator_spelling(node, ancestors) { + *halstead_maps + .primitive_operators + .entry(spelling.as_bytes()) + .or_insert(0) += 1; + } else if T::is_primitive(node) { // Store primitive-type operators by text so distinct // primitives (e.g. `int` vs `double`) that share a // single kind_id are counted separately in n1/N1. @@ -1278,9 +1286,11 @@ mod tests { // operators (one occurrence each). #1486 adds the // annotation `:` — one entry, 23 occurrences: `C`, the // two fields, `p`, the four params of `f`, and the - // fifteen `let` bindings. - assert_eq!(metric.halstead.unique_operators(), 34); - assert_eq!(metric.halstead.total_operators(), 144); + // fifteen `let` bindings. #1559 bills `Option`'s + // brackets as one `<>` pair: one entry and one + // occurrence fewer. + assert_eq!(metric.halstead.unique_operators(), 33); + assert_eq!(metric.halstead.total_operators(), 143); // u_operands / operands grew (was 31/50 before #390): the // fix now classifies TypeIdentifier (`T`, `S`, `Option`) // and FieldIdentifier (struct fields `x`, `y`) as operands @@ -1358,7 +1368,9 @@ mod tests { // Also covers issue #394: `::` is now an operator. The snippet // has two `::` tokens (`Vec::new`, `HashMap::new`), so n1 grew // from 10 → 11 and N1 from 17 → 19. #1486 then billed the two - // annotation `:` tokens as their own entry: 12 and 21. + // annotation `:` tokens as their own entry: 12 and 21. #1559 + // billed each generic's brackets as one `<>` pair, replacing the + // `<` and `>` entries and their four occurrences: 11 and 19. check_metrics::( "fn main() { let v: Vec = Vec::new(); @@ -1374,26 +1386,26 @@ mod tests { // `::` appears twice (Vec::new, HashMap::new); without // the #394 fix u_operators was 10 and operators 17. // `:` appears twice and stays distinct from `::`. - assert_eq!(metric.halstead.unique_operators(), 12); - assert_eq!(metric.halstead.total_operators(), 21); + assert_eq!(metric.halstead.unique_operators(), 11); + assert_eq!(metric.halstead.total_operators(), 19); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 12, - "total_operators": 21, + "unique_operators": 11, + "total_operators": 19, "unique_operands": 8, "total_operands": 11, - "length": 32, - "estimated_program_length": 67.01955000865388, - "purity_ratio": 2.0943609377704338, - "vocabulary": 20, - "volume": 138.3016990363956, - "difficulty": 8.25, - "level": 0.12121212121212122, - "effort": 1140.9890170502638, - "time": 63.38827872501466, - "bugs": 0.03639706698498207 + "length": 30, + "estimated_program_length": 62.05374780501027, + "purity_ratio": 2.068458260167009, + "vocabulary": 19, + "volume": 127.43782540330756, + "difficulty": 7.5625, + "level": 0.1322314049586777, + "effort": 963.7485546125134, + "time": 53.54158636736186, + "bugs": 0.03252279825177962 } "# ); @@ -1417,10 +1429,10 @@ mod tests { ("field pattern", "fn f() { let S { a: b } = s; }\n", [7, 8]), // `fn`, `()`, `:`, `u8`, `{}` ("parameter", "fn f(x: u8) {}\n", [5, 5]), - // `fn`, `<`, `:`, `>`, `()`, `{}` - ("trait bound", "fn f() {}\n", [6, 6]), - // `fn`, `<`, `>`, `()`, `where`, `:`, `{}` - ("where clause", "fn f() where T: Clone {}\n", [7, 7]), + // `fn`, `<>`, `:`, `()`, `{}` + ("trait bound", "fn f() {}\n", [5, 5]), + // `fn`, `<>`, `()`, `where`, `:`, `{}` + ("where clause", "fn f() where T: Clone {}\n", [6, 6]), // `fn`, `()`, `{}` twice, `:`, `loop`, `break`, `;` ("loop label", "fn f() { 'a: loop { break 'a; } }\n", [7, 8]), // `fn`, `()`, `{}`, `let`, `=`, `|` twice, `:`, `u8`, `;` @@ -1429,12 +1441,12 @@ mod tests { "fn f() { let c = |q: u8| q; }\n", [9, 10], ), - // `fn`, `()` twice, `{}`, `let`, `:`, `<`, `u8`, `>`, `=`, - // `::`, `;` — `:` and `::` are two entries + // `fn`, `()` twice, `{}`, `let`, `:`, `<>`, `u8`, `=`, `::`, + // `;` — `:` and `::` are two entries ( "beside a path", "fn f() { let v: Vec = Vec::new(); }\n", - [11, 12], + [10, 11], ), ]; assert_operator_rows::("foo.rs", ROWS); @@ -3443,17 +3455,17 @@ mod tests { assert_eq!(metric.halstead.total_operands(), 2); }); - // expected: operators `class`, `{`×2, `void`, `(`, `;`, `<`, - // `>`, `int` — 9 total, 8 unique (`int` is the text-keyed - // primitive operator, per #286). Operands `C`, `M`, `List`, `l` + // expected: operators `class`, `{`×2, `void`, `(`, `;`, `<>`, + // `int` — 8 total, 7 unique (`int` is the text-keyed primitive + // operator, per #286; `<>` is the type-argument pair, #1559). Operands `C`, `M`, `List`, `l` // — 4/4. Pre-fix the `generic_name` added `List`, making // them 5/5. check_metrics::( "class C { void M() { List l; } }", "foo.cs", |metric| { - assert_eq!(metric.halstead.unique_operators(), 8); - assert_eq!(metric.halstead.total_operators(), 9); + assert_eq!(metric.halstead.unique_operators(), 7); + assert_eq!(metric.halstead.total_operators(), 8); assert_eq!(metric.halstead.unique_operands(), 4); assert_eq!(metric.halstead.total_operands(), 4); }, @@ -6252,30 +6264,31 @@ end", ("enum class", "enum class F { B };\n", [4, 4]), // `struct`, `;` — an elaborated type bills its keyword ("elaborated type", "struct S s;\n", [2, 2]), - // `template`, `<`, `class`, `>`, `struct`, `{}`, `;` + // `template`, `<>`, `class`, `struct`, `{}`, `;` — the parameter + // list's brackets are one pair (#1559) ( "class parameter", "template struct W {};\n", - [7, 7], + [6, 6], ), // as above, `typename` in place of `class` ( "typename parameter", "template struct W {};\n", - [7, 7], + [6, 6], ), - // `template`, `<`, `class`, `=`, `int`, `>`, `struct`, `{}`, `;` + // `template`, `<>`, `class`, `=`, `int`, `struct`, `{}`, `;` ( "optional class parameter", "template struct W {};\n", - [9, 9], + [8, 8], ), - // `template`, `<`, `class`, `>`, `struct`, `{}`, `;` — `...` is - // an operand + // `template`, `<>`, `class`, `struct`, `{}`, `;` — `...` is an + // operand ( "variadic class parameter", "template struct W {};\n", - [7, 7], + [6, 6], ), ]; @@ -6410,12 +6423,12 @@ end", vararg c: Int) {}\n", [10, 16], ), - // `inline`, `reified`, `in`, `out`, `fun`, `class`, `<` x2, - // `>` x2, `,`, `()`, `{}` + // `inline`, `reified`, `in`, `out`, `fun`, `class`, `<>` x2, + // `,`, `()`, `{}` ( "type parameter", "inline fun f() {}\nclass Box\n", - [11, 13], + [10, 11], ), // `expect`, `actual`, `fun` x2, `()` x2, `{}` ("platform", "expect fun f()\nactual fun g() {}\n", [5, 7]), @@ -6467,19 +6480,19 @@ end", ("alias declaration", "using U = int;\n", [4, 4]), // `using`, `namespace`, `;` ("using directive", "using namespace std;\n", [3, 3]), - // `template`, `<`, `typename`, `>`, `;` and the `struct` of `S` + // `template`, `<>`, `typename`, `;` and the `struct` of `S` ( "template declaration", "template struct S;\n", - [6, 6], + [5, 5], ), // `typename`, `::`, `;` ("dependent type", "typename T::type x;\n", [3, 3]), - // `void`, `()` twice, `{}`, `.`, `template`, `<`, `int`, `>`, `;` + // `void`, `()` twice, `{}`, `.`, `template`, `<>`, `int`, `;` ( "template disambiguator", "void g() { x.template f(); }\n", - [9, 10], + [8, 9], ), ]; @@ -6590,11 +6603,11 @@ end", ("satisfies", "let s = w satisfies Sat;\n", [4, 4]), // `type`, `=`, `keyof`, `;` ("keyof", "type K = keyof T;\n", [4, 4]), - // `type`, `<` x2, `>` x2, `=`, `extends`, `infer`, `?`, `:`, `;` + // `type`, `<>` x2, `=`, `extends`, `infer`, `?`, `:`, `;` ( "infer", "type I = X extends Array ? E : X;\n", - [9, 11], + [8, 9], ), // `function`, `()`, `:` x2, `unknown`, `is`, `{}`, `return`, `;` ( diff --git a/src/ops.rs b/src/ops.rs index 62c9468f..b1e3f73e 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -1465,4 +1465,189 @@ mod tests { ], ); } + + /// #1559: a generic list's angle brackets are one `<>` bracket pair, + /// billed on the opener as `()` / `[]` / `{}` are, with the closer + /// billing nothing. Every fixture also spells a comparison `<` and + /// `>`, which must stay their own vocabulary entries: the grammars + /// give the list brackets the comparison operators' kind ids, so the + /// vocabulary is what shows the two apart and `N1` is what shows the + /// closer is not billed. + #[cfg(any( + feature = "cpp", + feature = "csharp", + feature = "groovy", + feature = "java", + feature = "kotlin", + feature = "mozcpp", + feature = "rust", + feature = "typescript", + ))] + fn check_generic_pair(lang: LANG, file: &str, source: &str, operators: &[&str], n1: [u64; 2]) { + let source = Source::new(lang, source.as_bytes()).with_name(Some(file.to_owned())); + let ast = Ast::parse(source).expect("language feature enabled"); + let ops = ast.ops().expect("ops walk must yield a top-level Ops"); + let mut expected = operators.to_vec(); + expected.sort_unstable(); + assert_eq!(ops.operators, expected, "{lang:?} vocabulary"); + + let space = ast + .metrics(crate::MetricsOptions::default()) + .expect("metrics walk"); + let halstead = &space.metrics.halstead; + assert_eq!( + [halstead.unique_operators(), halstead.total_operators()], + n1, + "{lang:?} [n1, N1]" + ); + } + + // expected: `<>` three times (one parameter list, two nested + // argument lists), never `>`; `<` and `>` once each, the comparisons. + #[cfg(feature = "rust")] + #[test] + fn rust_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Rust, + "foo.rs", + "fn f(v: Vec>) -> bool { a < b && c > d }\n", + &["fn", "<>", "()", ":", "->", "bool", "{}", "<", "&&", ">"], + [10, 12], + ); + // A turbofish and a higher-ranked `for<'a>` (`for_lifetimes`) + // are lists too, `<>` twice (`fn` is twice as well); the `>>` shift + // is its own operator. + check_generic_pair( + LANG::Rust, + "foo.rs", + "fn g(f: for<'a> fn(&'a u8)) { h::(x >> 1); }\n", + &[ + "fn", "()", ":", "for", "<>", "&", "u8", "{}", "::", ">>", ";", + ], + [11, 16], + ); + // A precise-capturing `use` (`use_bounds`) is a list as well. + check_generic_pair( + LANG::Rust, + "foo.rs", + "fn u(x: T) -> impl Sized + use { x }\n", + &["fn", "<>", "()", ":", "->", "impl", "+", "use", "{}"], + [9, 10], + ); + } + + // expected: `<>` three times, `;` and `{}` twice each. + #[cfg(feature = "java")] + #[test] + fn java_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Java, + "foo.java", + "class A { List> v; boolean m() { return a < b && c > d; } }\n", + &[ + "class", + "<>", + ";", + "{}", + "boolean_type", + "()", + "return", + "<", + "&&", + ">", + ], + [10, 14], + ); + } + + // expected: as Java, with `bool` for `boolean_type`. + #[cfg(feature = "csharp")] + #[test] + fn csharp_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Csharp, + "foo.cs", + "class A { List> v; bool M() { return a < b && c > d; } }\n", + &[ + "class", "<>", ";", "{}", "bool", "()", "return", "<", "&&", ">", + ], + [10, 14], + ); + } + + // expected: `<>` three times, `:` twice. TSX shares the getter macro + // and the same arm, so it is checked on the same input. + #[cfg(feature = "typescript")] + #[test] + fn typescript_generic_angles_are_one_pair_1559() { + const SOURCE: &str = + "function f(v: Array>): boolean { return a < b && c > d; }\n"; + const OPERATORS: &[&str] = &[ + "function", "<>", "()", ":", "boolean", "{}", "return", "<", "&&", ">", ";", + ]; + check_generic_pair(LANG::Typescript, "foo.ts", SOURCE, OPERATORS, [11, 14]); + check_generic_pair(LANG::Tsx, "foo.tsx", SOURCE, OPERATORS, [11, 14]); + } + + // expected: `<>` three times (the `template` parameter list and two + // nested argument lists), `::` twice. The closer of the outer list + // is spelled `>>` and still parses as two `>` tokens. + #[cfg(feature = "cpp")] + #[test] + fn cpp_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Cpp, + "foo.cpp", + "template bool f(std::vector> v) { return a < b && c > d; }\n", + &[ + "template", "<>", "class", "bool", "()", "::", "{}", "return", "<", "&&", ">", ";", + ], + [12, 15], + ); + } + + // expected: the Cpp row, through the Mozcpp grammar, which owns no + // file extension and so has no corpus coverage of its own. + #[cfg(feature = "mozcpp")] + #[test] + fn mozcpp_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Mozcpp, + "foo.cpp", + "template bool f(std::vector> v) { return a < b && c > d; }\n", + &[ + "template", "<>", "class", "bool", "()", "::", "{}", "return", "<", "&&", ">", ";", + ], + [12, 15], + ); + } + + // expected: `<>` three times, `:` twice. + #[cfg(feature = "kotlin")] + #[test] + fn kotlin_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Kotlin, + "foo.kt", + "fun f(v: List>): Boolean { return a < b && c > d }\n", + &["fun", "<>", "()", ":", "{}", "return", "<", "&&", ">"], + [9, 12], + ); + } + + // expected: `<>` four times — Groovy spells a method's parameter + // list as its own `method_type_parameters` node. + #[cfg(feature = "groovy")] + #[test] + fn groovy_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Groovy, + "foo.groovy", + "class A { List> v; def boolean m() { return a < b && c > d } }\n", + &[ + "class", "<>", ";", "{}", "def", "()", "return", "<", "&&", ">", + ], + [10, 14], + ); + } } diff --git a/tests/parity/abc_halstead_bracket_parity.rs b/tests/parity/abc_halstead_bracket_parity.rs index 14004c06..8aa80317 100644 --- a/tests/parity/abc_halstead_bracket_parity.rs +++ b/tests/parity/abc_halstead_bracket_parity.rs @@ -45,12 +45,14 @@ use super::support::{metrics_space, ops_space}; /// /// The wildcard arm is deliberate, unlike the exhaustive `match` in /// [`super::ops_metrics_space_parity::fixture`]. Eighteen languages -/// carry the ABC bracket gate — the generic and template brackets of -/// C++, Rust, Go, Java, Groovy and the rest are the same shape — so an -/// exhaustive table would need a row for each rather than a `None` arm -/// asserting they have no such construct, which would be false. What -/// this pins is the policy, and a policy flip lands in a shared getter -/// macro or a shared ABC arm, which these seven rows already reach. +/// carry the ABC bracket gate, so an exhaustive table would need a row +/// for each rather than a `None` arm asserting they have no such +/// construct, which would be false. What this pins is the policy, and a +/// policy flip lands in a shared getter macro or a shared ABC arm, +/// which these seven rows already reach. Generic and template brackets +/// are the one family left out on purpose: Halstead bills them as a +/// single `<>` pair since #1559, which the `*_generic_angles_*` tests in +/// `src/ops.rs` pin per language. // // Two gates, one condition. The `mod` gate in `main.rs` names the seven // languages with a `Some` row and is the one that decides whether any of diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index 58a78b26..a737c2ff 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit 58a78b269e18a6508890ae5609f0fbb44c719cd3 +Subproject commit a737c2ff02d8b9884700bf5e6d024c10e14ab9d1 From cbe6c58a91f420a9c4e15a87599df8ae19d15978 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:41:29 -0700 Subject: [PATCH 38/48] refactor(getter): move the JS-family macro out `impl_js_family_get_op_type!` and its per-language rationale are shared by exactly the four JS-family getter modules, yet sat in the middle of getter.rs between the trait and its helpers. #1483 and #1559 grew it (JSX text, generic angles), pushing getter.rs to 1,264 sloc, inside the 0.95 soft band of the 1,300 file limit. Move it verbatim to getter/js_family.rs, beside generic_angles.rs, and re-export it so the `use super::*` call sites resolve unchanged. getter.rs drops to ~970 sloc and leaves the baseline. The macro's three rustfmt-bail arms move with it, so the bail baseline relocates them rather than growing. No behaviour change. Fixes #1553 --- .bca-baseline.toml | 6 - .rustfmt-bail-baseline.txt | 2 +- big-code-analysis-ast/src/getter.rs | 297 +---------------- big-code-analysis-ast/src/getter/js_family.rs | 300 ++++++++++++++++++ 4 files changed, 303 insertions(+), 302 deletions(-) create mode 100644 big-code-analysis-ast/src/getter/js_family.rs diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 262cad40..39400b8d 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -221,12 +221,6 @@ qualified = "step_raw_string" metric = "nargs" value = 5.0 -[[entry]] -path = "big-code-analysis-ast/src/getter.rs" -qualified = "" -metric = "loc.sloc" -value = 1264.0 - [[entry]] path = "big-code-analysis-ast/src/getter.rs" qualified = "Getter::is_braced_literal_slot" diff --git a/.rustfmt-bail-baseline.txt b/.rustfmt-bail-baseline.txt index b14584e0..3d8cc96e 100644 --- a/.rustfmt-bail-baseline.txt +++ b/.rustfmt-bail-baseline.txt @@ -122,7 +122,6 @@ # not checking it, and it does drift: before #1136 `src/getter/java.rs` # carried a 252-character match arm. -big-code-analysis-ast/src/getter.rs 3 big-code-analysis-ast/src/getter/bash.rs 3 big-code-analysis-ast/src/getter/c.rs 3 big-code-analysis-ast/src/getter/cpp.rs 4 @@ -132,6 +131,7 @@ big-code-analysis-ast/src/getter/go.rs 1 big-code-analysis-ast/src/getter/groovy.rs 9 big-code-analysis-ast/src/getter/irules.rs 6 big-code-analysis-ast/src/getter/java.rs 6 +big-code-analysis-ast/src/getter/js_family.rs 3 big-code-analysis-ast/src/getter/kotlin.rs 4 big-code-analysis-ast/src/getter/lua.rs 3 big-code-analysis-ast/src/getter/mozcpp.rs 4 diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index 75d6d2dd..f9b53a9e 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -18,6 +18,7 @@ use crate::lang_helpers::tcl_family::{ use crate::space_kind::SpaceKind; use crate::traits::Search; use generic_angles::GenericAngleKinds; +use js_family::impl_js_family_get_op_type; use crate::*; @@ -67,301 +68,6 @@ macro_rules! get_operator { }; } -// Emit a `Getter::get_op_type` body for a JS-family language. The four -// JS-family grammars (JavaScript, MozJS, TypeScript, TSX) share most of -// their Halstead operator/operand kind classifications; per-language -// deltas are passed as bracketed extras so all four impls stay in -// lockstep when a kind is added or removed (issue #299). -// -// `$op_extras` per language: -// * JavaScript / MozJS: `OptionalChain` — the bare `?.` token (these -// grammars expose no `optional_chain` wrapper). -// * TypeScript / TSX: `QMARKDOT`, `PredefinedType` — `QMARKDOT` is -// the bare `?.` token under the `optional_chain` wrapper (issue -// #281); `PredefinedType` is the TS type keyword set (`string`, -// `number`, `boolean`, …). -// * JavaScript / MozJS / TSX: `LTSLASH`, `SLASHGT` — the JSX closing -// and self-closing tag delimiters, which these grammars spell as -// their own two-character tokens rather than as a `<` / `>` pair. -// Until #1395 neither was in either arm, so a JSX element billed -// its opening `<` and its `>`s but nothing for `` -// reported one bracket operator where the source spells two. They -// are extras rather than members of the shared arm because -// TypeScript is the one grammar of the four with no JSX and -// therefore no such variants to name. -// -// `$operand_extras` per language: -// * JavaScript / MozJS / TSX: `Identifier2`, `String2` — anonymous -// keyword aliases the JS grammar exposes for `Identifier` and -// `String`. TSX's `String2` (kind_id 261) is its string-literal -// alias, so it stays an operand like JS's. -// * TypeScript: none. TS's own aliases are either operators or -// deliberately unclassified (below). -// The `: string` type-keyword aliases (TS `String2`, kind_id 135; -// TSX `String3`, kind_id 141) are deliberately absent: they are -// emitted only as the child of a `predefined_type` wrapper, which -// already contributes the text-keyed `"string"` operator, and -// #313's listing of the keyword child as an operand counted one -// source token twice (#1261, which also narrowed -// `Checker::is_string` to match). -// -// **A member access contributes its leaves, never the composite.** -// `a.b` is the operands `a` and `b` plus the `.` operator — three -// vocabulary entries, not four. Until #1263 the operand arm listed the -// `member_expression` wrapper *and* the walker descended into its -// `object` / `property` children, so every access billed one extra -// `N2` entry keyed on the whole `a.b` text (and `a.b.c` billed two: -// `a.b` and `a.b.c`). The composite is what the rest of the workspace -// already excludes — C, C++, Java, Rust, Python, Go, Kotlin, Ruby, Lua -// and PHP were all measured leaves-only for the identical shape — so -// identical code scored differently by language. The same call applies -// to TS's `nested_identifier` (`namespace N.M`), C#'s -// `qualified_name` / `generic_name` / `alias_qualified_name`, and -// Groovy's `qualified_name` / `qualified_type`. -// -// This is grammar-dispatch section 5's "any predicate listing both a -// container and a kind it can contain double-counts", and its section -// 6 corollary is why `PrivatePropertyIdentifier` joins the operand arm -// in the same change: `this.#x`'s `#x` leaf was in no operand list, so -// the composite had been its only count, and a bare deletion would -// have regressed private-field access to zero operands. (The `#x` in -// the *field definition* `#x = 1` had never been counted at all — no -// wrapper covered it — so this fixes that half too.) -// -// `jsx: [, ]` (JavaScript, MozJS, TSX) bills -// the content of a JSX element (#1483). A `jsx_text` node is an -// operand, the way a string's contents are; it is childless and no -// node containing it (`jsx_element`) is classified, so nothing bills it -// twice. Two refinements need the source bytes and so live in the -// `get_op_type_with_code` / `get_operand_id` overrides the parameter -// also emits: -// -// * A whitespace-only `jsx_text` is `Unknown`. The scanner emits no -// token for whitespace that starts with a newline, but same-line -// spaces between tags (` `) are a node of their own, and -// billing them would make `N2` follow the source's layout. -// * The operand is keyed on the text with its surrounding whitespace -// trimmed. A token keeps the newlines and indentation around it, so -// `Start` at two indentation depths would otherwise be two distinct -// operands. This is the same shape as Kotlin's #454 narrowing. -// -// An `html_character_reference` (`&`) splits the text around it -// into siblings, so `a & b` is three operands: `a`, `&` and -// `b`. Each node is one vocabulary entry, and the entity denotes a -// character the surrounding text does not, so it is not folded into -// either neighbour. The same kind also sits inside a JSX attribute's -// `string`, which is already the operand, so it is billed only when -// its parent is not a string. -// -// `Checker::is_string` deliberately does not list `jsx_text`: `find -// string` reports delimited string literals, the set the alterator -// flattens (#283), and element text is undelimited markup content. -// -// The `TemplateString` interpolation guard is shared verbatim (issue -// #192): a bare `` `...` `` mirrors a `"..."` operand, but an -// interpolated template must yield `Unknown` because its inner -// `TemplateSubstitution` expressions are walked separately. -// -// The `Regex` arms are shared the same way (issue #1314), and for the -// same reason: all four grammars spell a regex literal identically — -// a `regex` wrapper whose two delimiters are `SLASH`, the kind id real -// division uses — so neither arm has a per-language delta to hoist to -// a call site. Only the ids differ (`SLASH` 87/81/90/87, `Regex` -// 224/250/264/225) and the macro names both through the enum, so no -// invocation mentions either. See the arms themselves for the -// fabrication and the missing-operand halves. -macro_rules! impl_js_family_get_op_type { - ( - $lang:ident, - op_extras: [$($op_extra:ident),* $(,)?], - operand_extras: [$($operand_extra:ident),* $(,)?] - $(, predefined_void: $predefined_type:ident)? - $(, generic_angles: $generic_angles:ident)? - $(, jsx: [$jsx_text:ident, $jsx_entity:ident])? $(,)? - ) => { - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { - use $lang::*; - - $( - if $generic_angles.is_closer(node, ancestors) { - return TokenRole::Unknown; - } - )? - - // TS/TSX only: a `void` return / parameter type is parsed as a - // `predefined_type` wrapper around an inner `void` token. Both - // the wrapper (routed through `is_primitive` into the text-keyed - // `primitive_operators` map as `"void"`) and the inner `Void` - // token (a standalone expression operator, e.g. `void 0`) would - // otherwise classify as operators, double-counting one source - // `void` as two Halstead operators (issue #453). Only `void` has - // an operator-kind child; the `string` keyword's child is an - // operand-kind alias, which collided in the other direction - // (operator + operand) until #1261 dropped it from the operand - // extras. Suppress the wrapper here and - // let the inner `Void` token carry the single operator, keeping - // the kind_id-keyed count consistent with expression `void 0` - // (the lesson-4 `n1 == dedupe(ops.operators)` invariant). - $( - if node.kind_id() == $predefined_type as u16 - && node - .child(0) - .is_some_and(|child| child.kind_id() == Void as u16) - { - return TokenRole::Unknown; - } - )? - - match node.kind_id().into() { - // Regex delimiter punctuation. A `regex` literal spells - // both of its delimiters with the same kind id real - // division uses, so `const a = /abc/g;` reported a `/` - // operator with no division in the source and n1/N1 - // counted the literal's punctuation as arithmetic - // (#1314, the JS-family sibling of Elixir #1256 and - // Ruby/Perl #1312). The `Regex` node itself is the - // operand (below), so the delimiters are suppressed - // exactly when their parent is that node — the - // compound-leaf guard of grammar-dispatch section 5. - // - // Parent, not ancestor, for correctness by - // construction rather than by observation: unlike - // Ruby's `#{…}`, a JS regex admits no nested - // expression at all (`regex_pattern` and `regex_flags` - // are leaves), so no fixture can tell this guard from - // an ancestor-scanning one. The halstead test - // `js_regex_delimiter_guard_is_parent_scoped_is_unobservable` - // records that, and pins the grammar property it rests - // on, rather than implying coverage the suite lacks. - // - // `SLASH2` — the aliased regex-start token every one of - // these grammars carries — needs no arm: it is absent - // from the operator list below, so it already lands on - // `Unknown`, which is what a delimiter should be. That - // is the one place this differs from Ruby's guard, - // where `SLASH2` had to be moved off the arithmetic arm. - SLASH - if ancestors.parent_has_kind(node, Regex as u16) => - { - TokenRole::Unknown - } - Export | Import | Import2 | Extends | DOT | From | LPAREN | COMMA | As | STAR - | GTGT | GTGTGT | COLON | Return | Delete | Throw | Break | Continue | If - | Else | Switch | Case | Default | Async | Do | For | In | Of | While | Try - | Catch | Finally | With | EQ | AT | AMPAMP | PIPEPIPE | PLUS | DASH | DASHDASH - | PLUSPLUS | SLASH | PERCENT | STARSTAR | PIPE | AMP | LTLT | TILDE | LT | LTEQ - | EQEQ | BANGEQ | GTEQ | GT | PLUSEQ | BANG | BANGEQEQ | EQEQEQ | DASHEQ - | STAREQ | SLASHEQ | PERCENTEQ | STARSTAREQ | GTGTEQ | GTGTGTEQ | LTLTEQ | AMPEQ - | CARET | CARETEQ | PIPEEQ | Yield | LBRACK | LBRACE | Await | QMARK - | QMARKQMARK | EQGT | DOTDOTDOT | New | Let | Var | Const | Function - | SEMI | Typeof | Instanceof | Void - // `Function` and `Class2` are the `function` / `class` - // keyword leaves (#1552). Their expression wrappers — - // `FunctionExpression` and the unsuffixed `Class` — - // stay unlisted, so `const f = function () {}` bills - // one `function`, not two (#1554). The leaf is the - // keeper because it exists in every spelling: the - // `async` and generator forms, and an ERROR-recovery - // parse that drops the wrapper. - | Class2 - // `get`/`set` accessor keywords are operators, matching the - // C# getter's `Get | Set | Init | Add | Remove` accessor arm. - // Before #695 the JS family classified them as operands, so - // the same accessor keyword landed in opposite Halstead - // groups across languages, skewing n1/n2 for accessor-heavy - // code (#695). - | Set | Get - $(| $op_extra)* => TokenRole::Operator, - // `Regex` is the literal's own node and contributes one - // operand, the way Ruby's `Regex` and Elixir's `Sigil` - // do. It was in neither arm before #1314, so `/abc/g` - // reached the vocabulary from *neither* side: a - // fabricated `/` operator and no operand at all. Its - // `regex_pattern` / `regex_flags` children stay - // unclassified, so the wrapper cannot double-count - // them, and no interpolation is possible inside a JS - // regex — hence a plain arm here rather than the - // `string_operand_type` dispatch the template literal - // below needs. - // `PrivatePropertyIdentifier` is the `#x` leaf of a - // private class field, in both its declaration - // (`#x = 1`) and its access (`this.#x`). See the - // leaves-not-composites note above the macro for why - // it had to be added in the same change that dropped - // `MemberExpression*`. - // - // `MetaProperty` is `import.meta` / `new.target`: one - // atomic operand, like `this`, whose `meta` / `target` - // leaves are anonymous tokens no arm classifies. It is - // the one composite the #1263 drop has to keep - // (grammar-dispatch §6) — without it the meta-object - // contributes no operand at all while `this.env.x` - // still yields three. - Identifier | PropertyIdentifier | PrivatePropertyIdentifier | MetaProperty - | String | Number | True | False | Null | This | Super | Undefined | Regex - $(| $operand_extra)* => TokenRole::Operand, - // A `` `...` `` is a string literal; without interpolation it - // mirrors `"..."` and contributes one operand. When it has a - // `TemplateSubstitution` child the inner expression is already - // walked and classified, so counting the wrapper too would - // double-count its contribution to `N2` (issue #192, same - // pattern as #183 C# / #191 Kotlin / #199 Perl). - TemplateString => { - Self::string_operand_type(node, &[TemplateSubstitution as u16]) - } - $( - $jsx_text => TokenRole::Operand, - $jsx_entity - if !ancestors - .parent(node) - .is_some_and(|p| matches!(p.kind_id().into(), String | String2)) => - { - TokenRole::Operand - } - )? - _ => TokenRole::Unknown, - } - } - - $( - fn get_operator_spelling<'a>( - node: &Node<'a>, - ancestors: Ancestors<'a, '_>, - ) -> Option<&'static str> { - $generic_angles.opener_spelling(node, ancestors) - } - )? - - $( - fn get_op_type_with_code<'a>( - node: &Node<'a>, - code: &[u8], - ancestors: Ancestors<'a, '_>, - ) -> TokenRole { - if node.kind_id() == $lang::$jsx_text as u16 - && code[node.start_byte()..node.end_byte()].trim_ascii().is_empty() - { - return TokenRole::Unknown; - } - Self::get_op_type(node, ancestors) - } - - fn get_operand_id<'a>( - node: &Node<'a>, - code: &'a [u8], - _ancestors: Ancestors<'a, '_>, - ) -> &'a [u8] { - let text = &code[node.start_byte()..node.end_byte()]; - if node.kind_id() == $lang::$jsx_text as u16 { - text.trim_ascii() - } else { - text - } - } - )? - }; -} - /// The default space name: the node's `name` field, else ``. /// /// A free function as well as the trait default so a language that needs @@ -1243,6 +949,7 @@ mod groovy; mod irules; mod java; mod javascript; +mod js_family; mod kotlin; mod lua; mod mozcpp; diff --git a/big-code-analysis-ast/src/getter/js_family.rs b/big-code-analysis-ast/src/getter/js_family.rs new file mode 100644 index 00000000..32da5887 --- /dev/null +++ b/big-code-analysis-ast/src/getter/js_family.rs @@ -0,0 +1,300 @@ +//! The `get_op_type` template the four JS-family grammars share +//! (JavaScript, MozJS, TypeScript, TSX), kept apart from the trait so +//! the per-language deltas and their rationale read as one unit. + +// Emit a `Getter::get_op_type` body for a JS-family language. The four +// JS-family grammars (JavaScript, MozJS, TypeScript, TSX) share most of +// their Halstead operator/operand kind classifications; per-language +// deltas are passed as bracketed extras so all four impls stay in +// lockstep when a kind is added or removed (issue #299). +// +// `$op_extras` per language: +// * JavaScript / MozJS: `OptionalChain` — the bare `?.` token (these +// grammars expose no `optional_chain` wrapper). +// * TypeScript / TSX: `QMARKDOT`, `PredefinedType` — `QMARKDOT` is +// the bare `?.` token under the `optional_chain` wrapper (issue +// #281); `PredefinedType` is the TS type keyword set (`string`, +// `number`, `boolean`, …). +// * JavaScript / MozJS / TSX: `LTSLASH`, `SLASHGT` — the JSX closing +// and self-closing tag delimiters, which these grammars spell as +// their own two-character tokens rather than as a `<` / `>` pair. +// Until #1395 neither was in either arm, so a JSX element billed +// its opening `<` and its `>`s but nothing for `` +// reported one bracket operator where the source spells two. They +// are extras rather than members of the shared arm because +// TypeScript is the one grammar of the four with no JSX and +// therefore no such variants to name. +// +// `$operand_extras` per language: +// * JavaScript / MozJS / TSX: `Identifier2`, `String2` — anonymous +// keyword aliases the JS grammar exposes for `Identifier` and +// `String`. TSX's `String2` (kind_id 261) is its string-literal +// alias, so it stays an operand like JS's. +// * TypeScript: none. TS's own aliases are either operators or +// deliberately unclassified (below). +// The `: string` type-keyword aliases (TS `String2`, kind_id 135; +// TSX `String3`, kind_id 141) are deliberately absent: they are +// emitted only as the child of a `predefined_type` wrapper, which +// already contributes the text-keyed `"string"` operator, and +// #313's listing of the keyword child as an operand counted one +// source token twice (#1261, which also narrowed +// `Checker::is_string` to match). +// +// **A member access contributes its leaves, never the composite.** +// `a.b` is the operands `a` and `b` plus the `.` operator — three +// vocabulary entries, not four. Until #1263 the operand arm listed the +// `member_expression` wrapper *and* the walker descended into its +// `object` / `property` children, so every access billed one extra +// `N2` entry keyed on the whole `a.b` text (and `a.b.c` billed two: +// `a.b` and `a.b.c`). The composite is what the rest of the workspace +// already excludes — C, C++, Java, Rust, Python, Go, Kotlin, Ruby, Lua +// and PHP were all measured leaves-only for the identical shape — so +// identical code scored differently by language. The same call applies +// to TS's `nested_identifier` (`namespace N.M`), C#'s +// `qualified_name` / `generic_name` / `alias_qualified_name`, and +// Groovy's `qualified_name` / `qualified_type`. +// +// This is grammar-dispatch section 5's "any predicate listing both a +// container and a kind it can contain double-counts", and its section +// 6 corollary is why `PrivatePropertyIdentifier` joins the operand arm +// in the same change: `this.#x`'s `#x` leaf was in no operand list, so +// the composite had been its only count, and a bare deletion would +// have regressed private-field access to zero operands. (The `#x` in +// the *field definition* `#x = 1` had never been counted at all — no +// wrapper covered it — so this fixes that half too.) +// +// `jsx: [, ]` (JavaScript, MozJS, TSX) bills +// the content of a JSX element (#1483). A `jsx_text` node is an +// operand, the way a string's contents are; it is childless and no +// node containing it (`jsx_element`) is classified, so nothing bills it +// twice. Two refinements need the source bytes and so live in the +// `get_op_type_with_code` / `get_operand_id` overrides the parameter +// also emits: +// +// * A whitespace-only `jsx_text` is `Unknown`. The scanner emits no +// token for whitespace that starts with a newline, but same-line +// spaces between tags (` `) are a node of their own, and +// billing them would make `N2` follow the source's layout. +// * The operand is keyed on the text with its surrounding whitespace +// trimmed. A token keeps the newlines and indentation around it, so +// `Start` at two indentation depths would otherwise be two distinct +// operands. This is the same shape as Kotlin's #454 narrowing. +// +// An `html_character_reference` (`&`) splits the text around it +// into siblings, so `a & b` is three operands: `a`, `&` and +// `b`. Each node is one vocabulary entry, and the entity denotes a +// character the surrounding text does not, so it is not folded into +// either neighbour. The same kind also sits inside a JSX attribute's +// `string`, which is already the operand, so it is billed only when +// its parent is not a string. +// +// `Checker::is_string` deliberately does not list `jsx_text`: `find +// string` reports delimited string literals, the set the alterator +// flattens (#283), and element text is undelimited markup content. +// +// The `TemplateString` interpolation guard is shared verbatim (issue +// #192): a bare `` `...` `` mirrors a `"..."` operand, but an +// interpolated template must yield `Unknown` because its inner +// `TemplateSubstitution` expressions are walked separately. +// +// The `Regex` arms are shared the same way (issue #1314), and for the +// same reason: all four grammars spell a regex literal identically — +// a `regex` wrapper whose two delimiters are `SLASH`, the kind id real +// division uses — so neither arm has a per-language delta to hoist to +// a call site. Only the ids differ (`SLASH` 87/81/90/87, `Regex` +// 224/250/264/225) and the macro names both through the enum, so no +// invocation mentions either. See the arms themselves for the +// fabrication and the missing-operand halves. +macro_rules! impl_js_family_get_op_type { + ( + $lang:ident, + op_extras: [$($op_extra:ident),* $(,)?], + operand_extras: [$($operand_extra:ident),* $(,)?] + $(, predefined_void: $predefined_type:ident)? + $(, generic_angles: $generic_angles:ident)? + $(, jsx: [$jsx_text:ident, $jsx_entity:ident])? $(,)? + ) => { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { + use $lang::*; + + $( + if $generic_angles.is_closer(node, ancestors) { + return TokenRole::Unknown; + } + )? + + // TS/TSX only: a `void` return / parameter type is parsed as a + // `predefined_type` wrapper around an inner `void` token. Both + // the wrapper (routed through `is_primitive` into the text-keyed + // `primitive_operators` map as `"void"`) and the inner `Void` + // token (a standalone expression operator, e.g. `void 0`) would + // otherwise classify as operators, double-counting one source + // `void` as two Halstead operators (issue #453). Only `void` has + // an operator-kind child; the `string` keyword's child is an + // operand-kind alias, which collided in the other direction + // (operator + operand) until #1261 dropped it from the operand + // extras. Suppress the wrapper here and + // let the inner `Void` token carry the single operator, keeping + // the kind_id-keyed count consistent with expression `void 0` + // (the lesson-4 `n1 == dedupe(ops.operators)` invariant). + $( + if node.kind_id() == $predefined_type as u16 + && node + .child(0) + .is_some_and(|child| child.kind_id() == Void as u16) + { + return TokenRole::Unknown; + } + )? + + match node.kind_id().into() { + // Regex delimiter punctuation. A `regex` literal spells + // both of its delimiters with the same kind id real + // division uses, so `const a = /abc/g;` reported a `/` + // operator with no division in the source and n1/N1 + // counted the literal's punctuation as arithmetic + // (#1314, the JS-family sibling of Elixir #1256 and + // Ruby/Perl #1312). The `Regex` node itself is the + // operand (below), so the delimiters are suppressed + // exactly when their parent is that node — the + // compound-leaf guard of grammar-dispatch section 5. + // + // Parent, not ancestor, for correctness by + // construction rather than by observation: unlike + // Ruby's `#{…}`, a JS regex admits no nested + // expression at all (`regex_pattern` and `regex_flags` + // are leaves), so no fixture can tell this guard from + // an ancestor-scanning one. The halstead test + // `js_regex_delimiter_guard_is_parent_scoped_is_unobservable` + // records that, and pins the grammar property it rests + // on, rather than implying coverage the suite lacks. + // + // `SLASH2` — the aliased regex-start token every one of + // these grammars carries — needs no arm: it is absent + // from the operator list below, so it already lands on + // `Unknown`, which is what a delimiter should be. That + // is the one place this differs from Ruby's guard, + // where `SLASH2` had to be moved off the arithmetic arm. + SLASH + if ancestors.parent_has_kind(node, Regex as u16) => + { + TokenRole::Unknown + } + Export | Import | Import2 | Extends | DOT | From | LPAREN | COMMA | As | STAR + | GTGT | GTGTGT | COLON | Return | Delete | Throw | Break | Continue | If + | Else | Switch | Case | Default | Async | Do | For | In | Of | While | Try + | Catch | Finally | With | EQ | AT | AMPAMP | PIPEPIPE | PLUS | DASH | DASHDASH + | PLUSPLUS | SLASH | PERCENT | STARSTAR | PIPE | AMP | LTLT | TILDE | LT | LTEQ + | EQEQ | BANGEQ | GTEQ | GT | PLUSEQ | BANG | BANGEQEQ | EQEQEQ | DASHEQ + | STAREQ | SLASHEQ | PERCENTEQ | STARSTAREQ | GTGTEQ | GTGTGTEQ | LTLTEQ | AMPEQ + | CARET | CARETEQ | PIPEEQ | Yield | LBRACK | LBRACE | Await | QMARK + | QMARKQMARK | EQGT | DOTDOTDOT | New | Let | Var | Const | Function + | SEMI | Typeof | Instanceof | Void + // `Function` and `Class2` are the `function` / `class` + // keyword leaves (#1552). Their expression wrappers — + // `FunctionExpression` and the unsuffixed `Class` — + // stay unlisted, so `const f = function () {}` bills + // one `function`, not two (#1554). The leaf is the + // keeper because it exists in every spelling: the + // `async` and generator forms, and an ERROR-recovery + // parse that drops the wrapper. + | Class2 + // `get`/`set` accessor keywords are operators, matching the + // C# getter's `Get | Set | Init | Add | Remove` accessor arm. + // Before #695 the JS family classified them as operands, so + // the same accessor keyword landed in opposite Halstead + // groups across languages, skewing n1/n2 for accessor-heavy + // code (#695). + | Set | Get + $(| $op_extra)* => TokenRole::Operator, + // `Regex` is the literal's own node and contributes one + // operand, the way Ruby's `Regex` and Elixir's `Sigil` + // do. It was in neither arm before #1314, so `/abc/g` + // reached the vocabulary from *neither* side: a + // fabricated `/` operator and no operand at all. Its + // `regex_pattern` / `regex_flags` children stay + // unclassified, so the wrapper cannot double-count + // them, and no interpolation is possible inside a JS + // regex — hence a plain arm here rather than the + // `string_operand_type` dispatch the template literal + // below needs. + // `PrivatePropertyIdentifier` is the `#x` leaf of a + // private class field, in both its declaration + // (`#x = 1`) and its access (`this.#x`). See the + // leaves-not-composites note above the macro for why + // it had to be added in the same change that dropped + // `MemberExpression*`. + // + // `MetaProperty` is `import.meta` / `new.target`: one + // atomic operand, like `this`, whose `meta` / `target` + // leaves are anonymous tokens no arm classifies. It is + // the one composite the #1263 drop has to keep + // (grammar-dispatch §6) — without it the meta-object + // contributes no operand at all while `this.env.x` + // still yields three. + Identifier | PropertyIdentifier | PrivatePropertyIdentifier | MetaProperty + | String | Number | True | False | Null | This | Super | Undefined | Regex + $(| $operand_extra)* => TokenRole::Operand, + // A `` `...` `` is a string literal; without interpolation it + // mirrors `"..."` and contributes one operand. When it has a + // `TemplateSubstitution` child the inner expression is already + // walked and classified, so counting the wrapper too would + // double-count its contribution to `N2` (issue #192, same + // pattern as #183 C# / #191 Kotlin / #199 Perl). + TemplateString => { + Self::string_operand_type(node, &[TemplateSubstitution as u16]) + } + $( + $jsx_text => TokenRole::Operand, + $jsx_entity + if !ancestors + .parent(node) + .is_some_and(|p| matches!(p.kind_id().into(), String | String2)) => + { + TokenRole::Operand + } + )? + _ => TokenRole::Unknown, + } + } + + $( + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + $generic_angles.opener_spelling(node, ancestors) + } + )? + + $( + fn get_op_type_with_code<'a>( + node: &Node<'a>, + code: &[u8], + ancestors: Ancestors<'a, '_>, + ) -> TokenRole { + if node.kind_id() == $lang::$jsx_text as u16 + && code[node.start_byte()..node.end_byte()].trim_ascii().is_empty() + { + return TokenRole::Unknown; + } + Self::get_op_type(node, ancestors) + } + + fn get_operand_id<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + ) -> &'a [u8] { + let text = &code[node.start_byte()..node.end_byte()]; + if node.kind_id() == $lang::$jsx_text as u16 { + text.trim_ascii() + } else { + text + } + } + )? + }; +} + +pub(super) use impl_js_family_get_op_type; From 35fd6c37f19d9b1878f289fce4f30a05b06dc97f Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:47:26 -0700 Subject: [PATCH 39/48] docs(changelog): consolidate entries from batch fix Record the behaviour changes from the 2026-10-09 batch under [Unreleased]: the Halstead vocabulary policy changes under Changed, the new defaulted Getter::get_operator_spelling under Added, and the per-language metric fixes under Fixed. --- CHANGELOG.md | 138 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7cae698..3bc96467 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,77 @@ cognitive; the word-operator entry (#1530) moves cognitive only; and the Elixir clause entry (#1531) moves cyclomatic for a typespec function type and a keyword-form `cond` catch-all. +**Halstead drift.** The entries under **Changed** apply #1395's rule that +punctuation and declaration keywords are vocabulary to the languages +that still left them out, so Halstead and the Maintainability Index +move for most Python, Rust, C, C++, Java, JavaScript, TypeScript, +Kotlin, Groovy and C# code. Python and Rust rise most, from #1486. The +generic `<>` pair (#1559) lowers effort for most code that uses +generics. Several **Fixed** entries also move only Halstead and are +marked as such. + +### Added + +- **`Getter::get_operator_spelling`** (#1559), a defaulted trait method + that lets a getter bill an operator under a fixed text key when its + grammar gives two operators the same kind id. It is additive: existing + implementations compile unchanged. + +### Changed + +- **Python and Rust bill their punctuation as Halstead operators** + (#1486). Python bills `()`, `[]` and `{}` once per pair, plus `;` and + every syntactic `:`. Rust bills `:` as an entry separate from `::`. An + f-string's `{` and format-spec `:` stay unbilled. Every Python and Rust + Halstead score rises and every MI score falls; on DeepSpeech's Python, + N1 rises by about 47%. +- **Type-declaration keywords are Halstead operators** in Java (`class`, + `interface`, `enum`, `record`, `@interface`), JavaScript and TypeScript + (`class`; TypeScript also `interface`, `enum`), Groovy (`class`, + `interface`, `enum`, `trait`, `record`), C and C++ (`struct`, `union`, + `enum`, C++ `class`), Objective-C (`@interface`, `@implementation`, + `@protocol`) and Python (`class`) (#1550, #1552). These languages now + match C#, Kotlin, PHP, Ruby and Rust. A `class` that declares nothing + (Java `C.class`) is not billed. +- **C++ and TypeScript declaration keywords and TypeScript type names** + (#1557). C++ bills `namespace`, `template`, `typename`, `typedef` and + `using`; C and Objective-C bill `typedef`. TypeScript bills `type`, + `namespace`, `module`, `declare` and `global`. Every TypeScript + `type_identifier`, including alias and interface names, annotations + and generic arguments, is now an operand. `template ` and + `template ` bill alike. +- **Java type names are Halstead operands** (#1560), as in C#, Rust and + TypeScript. A class literal's interior and the inferred `var` stay + unbilled. +- **Kotlin bills its declaration modifiers and `interface`** (#1556, + #1558): `enum`, `data`, `sealed`, the visibility keywords, `open`, + `override`, `suspend`, `reified`, variance `out` and the rest. Before + this, only `class` and `object` were billed. +- **TypeScript bills `satisfies`, `keyof`, `infer`, `is` and + `asserts`** (#1561). +- **Generic angle brackets bill as one `<>` pair** (#1559) in Rust (type + arguments and parameters, turbofish, `for<'a>`, `use`), Java, + Groovy, C#, TypeScript, TSX, Kotlin and C++ templates. The opening `<` + bills `<>` and the closing `>` bills nothing, the same convention as + `()`, `[]` and `{}`. Comparisons and shifts are unchanged. N1 falls + for code that uses generics, and effort fell in about 87% of the moved + C++ and Rust corpus files. +- **Perl and PHP `xor` scores as a value, not a decision** (#1536), like + `^` and the eager `xor` of Java, C++ and Kotlin. It adds no + cyclomatic or cognitive increment, and the boolean slot holding it + pays one ABC condition. PHP `xor` moves in all three metrics. +- **Rust `let`-`else` is a decision** (#1542, #1548). It adds one + cyclomatic decision (standard and modified), and cognitive scores it + +1 plus nesting, the same as its `if let` twin. Before, cyclomatic + ignored it and cognitive gave a flat +1 through the `else` token. +- **Elixir `with` clauses are decisions** (#1535). Each `pattern <- expr` + adds one cyclomatic decision and one ABC condition, like Rust's `if + let`, and a `when` guard on the pattern counts as a guard. Modified + cyclomatic no longer charges `with` a flat +1; its `else` adds one only + when it holds a clause other than `_`. A `for … reduce:` accumulator + clause follows the multi-clause `fn` rule, so a single clause is no + longer a decision. + ### Fixed - **A C++ or Mozcpp comparison-operator overload no longer scores a @@ -199,6 +270,73 @@ type and a keyword-form `cond` catch-all. `and` now keys to `&&` and `or` to `||`, as C++ does since #1522. Perl's low-precedence `and`, which cognitive never counted because the grammar parses it as a `unary_expression`, now scores like `&&`. +- **A negated C# call argument scores a condition** (#1537). `G(!a)`, + `G(name: !a)`, `new B(!a)`, `: base(!a)` and primary-constructor base + calls scored 0, where Java's `g(!a)` scores 1, because the `argument` + wrapper was never peeled. +- **Perl comparisons no longer over-count a condition when the grammar + mis-binds precedence** (#1273). tree-sitter-perl 1.1.2 parses + `$a > $b ? 1 : 2` as `$a > ($b ? 1 : 2)`, and `$a > $b && $c` the same + way. Each misparsed chaining comparison (`< > <= >= lt gt le ge == != + eq ne`) added one ABC condition. They now score like their + parenthesised and JavaScript twins. This is a local workaround for an + upstream grammar defect. +- **Perl auto-quoted hash keys no longer score as operators** (#1539, + #1541, #1545). tree-sitter-perl 1.1.2 misparses bareword keys: + - `and` (`$h{and}`, `(and => 1)`) scored a decision in cyclomatic, + cognitive and ABC. + - `not` was billed as a Halstead operator and peeled by ABC as a + negation. + - A `-bareword` key (`$h{-foo}`, `(-text => 1)`) was read as a file + test, which billed a fragment of the word (`oo`) and an ABC branch. + Each key now scores like its quoted or `or` twin, and real operators + and file tests are unchanged. +- **Perl `$x =~ s///` and `tr///` no longer score a condition by use** + (#1540), matching bare `s///`. A substitution is an edit, not a test. + Inside a boolean slot it still counts once, and pattern matches are + unchanged. +- **Perl Halstead no longer bills comment text** (#1549). The `#` of + every comment was billed as an operator, and so were the `#`, `[`, `]` + and `\` inside a bare `/…/` pattern, which now scores as `m//` and + `qr//` do. Halstead only. +- **Bash `-a` / `-o` and `=` score like their `&&` / `==` twins** + (#1536). `[ X -a Y ]`, `[ X ] && [ Y ]` and `[[ X && Y ]]` now agree + in cyclomatic, ABC and cognitive. `-a` / `-o` are logical connectives, + not ABC comparisons, and `=` inside a test is an ABC comparison. It + stays an assignment in `(( … ))`. +- **An Elixir named operator is billed once** (#1534). `&==/2`, + `&and/2` and `Kernel.||(a, b)` billed both an operand and an operator, + and a named `&&` / `||` / `and` / `or` added a cyclomatic decision. +- **C# lambda modifiers and implicit parameters are billed** (#1482, + #1544). A lambda's or anonymous method's `static` / `async` is an + operator, keyed as the declaration spelling is. The bare parameter of + `x => x` is an operand, as in `(x) => x`. Halstead only. +- **Overloaded operator names are Halstead operators** (#1296). This + covers C# `operator true` / `operator false` (previously operands), + and C++ / Mozcpp `operator[]`, `operator()`, `operator""_x`, + `operator co_await` and the `operator` keyword (previously billed + nowhere). `co_await`, `co_return` and `co_yield` in an expression are + operators too, and NExits counts `co_return` and `co_yield` as exits + (#1547). +- **JSX element text is a Halstead operand** (#1483), keyed on its + trimmed text, in JavaScript, Mozjs and TSX. Whitespace-only text is + ignored, and a character reference such as `&` in element content + is its own operand. +- **A JavaScript function expression bills `function` once** (#1554), + not twice. This applies to JavaScript, Mozjs, TypeScript and TSX. +- **Python f-string `=` is not an assignment** (#1551). The + self-documenting `=` of `f"{x=}"` is no longer billed. Halstead only. +- **Halstead skips zero-width nodes from error recovery** (#1546). An + inserted identifier billed an operand spelled `""` in every language, + and an inserted `;` / `)` billed a second operator. +- **A bodiless C++ specifier opens no space** (#1555). Forward + declarations, elaborated type specifiers (`struct S *p;`, parameter + types, casts) and explicit template instantiations opened phantom + struct/class spaces. In DeepSpeech, 4,452 such spaces disappear. This + lowers `cyclomatic.sum` by one base per removed space. +- **`make check-test-lang-gates` catches a helper gated out of a build + its caller compiles in** (#1562), when the caller names its parsers + only inside an inner `#[cfg]` block. ## [2.3.0] - 2026-10-07 From 8e956cf4fce6cb039b39a0b6f612bea64c649ccd Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:09:10 -0700 Subject: [PATCH 40/48] fix(getter/objc): bill generic angle brackets as <> #1559 paired the generic angle brackets in every getter with generics but Objective-C, whose lists still billed `<` and `>` under the comparison operators' entries. The ObjC getter now carries a GENERIC_ANGLES table over its four generic lists: generic_specifier (`NSArray`), protocol_reference_list (`id`), parameterized_arguments (`@interface A : NSObject `), and the call argument_list's `Type<...>` form (`f(NSArray)`). The call form is the ArgumentList2 alias; ArgumentList is the preprocessor call's, which holds no angle brackets. A regression test covers all four lists, a nested generic and surviving comparisons; the book and changelog now list Objective-C. Two doc comments still described `xor` as a boolean-sequence operator after #1536 made it a value: the compute_booleans_keyed doc and the PHP word-form test comment in cognitive.rs. The PHP keyword-walker regression note in abc.rs is corrected the same way. Refs #1559 Refs #1536 --- CHANGELOG.md | 3 ++- big-code-analysis-ast/src/getter/objc.rs | 34 ++++++++++++++++++++++++ big-code-analysis-book/src/metrics.md | 5 +++- src/metrics/abc.rs | 6 ++--- src/metrics/cognitive.rs | 10 ++++--- src/ops.rs | 33 +++++++++++++++++++++++ 6 files changed, 82 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bc96467..2770f260 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,7 +84,8 @@ marked as such. `asserts`** (#1561). - **Generic angle brackets bill as one `<>` pair** (#1559) in Rust (type arguments and parameters, turbofish, `for<'a>`, `use`), Java, - Groovy, C#, TypeScript, TSX, Kotlin and C++ templates. The opening `<` + Groovy, C#, TypeScript, TSX, Kotlin, C++ templates and Objective-C + (lightweight generics, protocol lists and `id

`). The opening `<` bills `<>` and the closing `>` bills nothing, the same convention as `()`, `[]` and `{}`. Comparisons and shifts are unchanged. N1 falls for code that uses generics, and effort fell in about 87% of the moved diff --git a/big-code-analysis-ast/src/getter/objc.rs b/big-code-analysis-ast/src/getter/objc.rs index 1960fce7..a5ff376b 100644 --- a/big-code-analysis-ast/src/getter/objc.rs +++ b/big-code-analysis-ast/src/getter/objc.rs @@ -4,6 +4,29 @@ use super::*; use crate::c_declarator::declarator_name; +// ObjC has no templates, but four nodes delimit a generic list with a +// bare `<` / `>`: `generic_specifier` (`NSArray`), +// `protocol_reference_list` (`id`), `parameterized_arguments` +// (`@interface A : NSObject `), and `argument_list`, whose +// `Type<…>` alternative (`f(NSArray)`) holds the brackets as +// direct children. A comparison's `<` / `>` is always the child of a +// `binary_expression` or `preproc_binary_expression`, so none of the +// four can claim one. `LT2` is the +// `token.immediate` `<` that `argument_list`'s form opens with; the +// parser reports it as `LT`, so it is listed defensively. The call's +// `argument_list` is `ArgumentList2`; `ArgumentList` is +// `preproc_call_expression`'s alias, which holds no angle brackets. +const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { + lists: &[ + Objc::GenericSpecifier as u16, + Objc::ProtocolReferenceList as u16, + Objc::ParameterizedArguments as u16, + Objc::ArgumentList2 as u16, + ], + openers: &[Objc::LT as u16, Objc::LT2 as u16], + closers: &[Objc::GT as u16], +}; + impl Getter for ObjcCode { fn get_func_space_name<'a, 'tree>( node: &Node<'tree>, @@ -70,6 +93,10 @@ impl Getter for ObjcCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Objc::*; + if GENERIC_ANGLES.is_closer(node, ancestors) { + return TokenRole::Unknown; + } + // ObjC is C plus message sends, blocks, and the `@`-directives. // The operator alphabet is therefore the C set (`src/getter.rs` // `impl Getter for CCode`) extended with the ObjC structural @@ -126,5 +153,12 @@ impl Getter for ObjcCode { } } + fn get_operator_spelling<'a>( + node: &Node<'a>, + ancestors: Ancestors<'a, '_>, + ) -> Option<&'static str> { + GENERIC_ANGLES.opener_spelling(node, ancestors) + } + get_operator!(Objc); } diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 796bb44f..88b58adc 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -744,7 +744,10 @@ tokens involved the same way they spell real operators: `h::()`, `for<'a>` and `use` in Rust; C++'s `template ` and `std::vector`; and the type-argument and type-parameter lists of Java, Groovy, C#, Kotlin, TypeScript and - TSX. A comparison's `<` and `>` and a shift's `<<` and `>>` stay + TSX. Objective-C has four generic lists: `NSArray`, + `id`, the protocol list in + `@interface A : NSObject `, and a type passed as a call + argument, `f(NSArray)`. A comparison's `<` and `>` and a shift's `<<` and `>>` stay separate operators, and a C++ `>>` that closes two template lists parses as two `>`, so it bills nothing. Go's generics use square brackets and already billed `[]`. Angle brackets around anything diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index 526a9bb9..28b5114d 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -6194,9 +6194,9 @@ function f(int $a, int $b): int { fn php_low_precedence_keyword_logical_ops_trigger_walker() { // Regression: pre-fix, `$a or $b` reported 0 conditions // because the dispatcher only handled `AMPAMP|PIPEPIPE`, - // skipping the PHP-specific `and` / `or` / `xor` keyword - // forms even though they parse under the same - // `binary_expression` shape. + // skipping the PHP-specific `and` / `or` keyword forms even + // though they parse under the same `binary_expression` shape. + // (`xor` is no chain: it is a value since #1536.) check_metrics::( " bool>(node: &Node, stats: &mut Stats, is_ /// changes which operator nests under which, and the parse already /// carries that, so `a || b and c` is `(a || b) and c` and still /// switches operation. Operators that are a different operation -/// (`xor`, `??`, `//`) keep keys of their own (#1530). +/// (`??`, `//`) keep keys of their own (#1530); `xor`, like `^`, is no +/// sequence operator at all (#1536). fn compute_booleans_keyed Option>( node: &Node, stats: &mut Stats, @@ -8059,11 +8060,12 @@ end", // Companions to `php_null_coalescing_cognitive_230`: the PHP // cognitive operator set extends past `&&` / `||` / `??` to include - // the word-form `and` / `or` / `xor`, mirroring PHP cyclomatic. A - // chain of identical word-form operators collapses to a single + // the word-form `and` / `or`, mirroring PHP cyclomatic. A chain of + // identical word-form operators collapses to a single // boolean-sequence increment under Sonar B1, the same way `&&` / // `||` chains do. Each word-form gets its own test so a regression - // that drops a single variant (e.g. only `Or`) is still caught. + // that drops a single variant (e.g. only `Or`) is still caught; + // `xor` is a value, not a sequence (#1536), and its test pins that. #[cfg(feature = "php")] #[test] diff --git a/src/ops.rs b/src/ops.rs index b1e3f73e..5dd491f5 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -1480,6 +1480,7 @@ mod tests { feature = "java", feature = "kotlin", feature = "mozcpp", + feature = "objc", feature = "rust", feature = "typescript", ))] @@ -1650,4 +1651,36 @@ mod tests { [10, 14], ); } + + // expected: `<>` five times, once per list — `parameterized_arguments` + // (`: NSObject `), `protocol_reference_list` + // (`id`), two nested `generic_specifier`s, and the + // `argument_list` type form `g(NSArray)` — and never `>`. + // `*` four times, `()` and `||` twice; `@end` and `id` bill nothing. + // Without the pair, `<` and `>` would each bill six times: [10, 25]. + #[cfg(feature = "objc")] + #[test] + fn objc_generic_angles_are_one_pair_1559() { + check_generic_pair( + LANG::Objc, + "foo.m", + "@interface A : NSObject \n@end\n\ + id f(NSArray *> *v) \ + { return g(NSArray) || a < b || c > d; }\n", + &[ + "@interface", + ":", + "<>", + "()", + "*", + "{}", + "return", + "||", + "<", + ">", + ";", + ], + [11, 20], + ); + } } From 7557c1645dc13c3e4fb4f59c6cb4a285dde03c1d Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:34:05 -0700 Subject: [PATCH 41/48] docs(lessons): record the batch's recovery and harness traps Add lesson 95: error recovery inserts zero-width tokens that is_missing() does not flag, on valid code the grammar cannot parse as well as invalid code (#1546). Extend lesson 35 with the #1540 token arm narrowed without its slot predicate. Route the standing obligations to rules: skip invented tokens by width (grammar-dispatch section 6), re-run perturbations after merging predicates and clear __pycache__ before a Python sweep (testing.md), and the three insta traps in the bulk snapshot refresh (AGENTS.md). --- .claude/rules/grammar-dispatch.md | 13 ++++++++ .claude/rules/testing.md | 20 +++++++++++++ AGENTS.md | 17 +++++++++++ docs/development/lessons_learned.md | 46 +++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+) diff --git a/.claude/rules/grammar-dispatch.md b/.claude/rules/grammar-dispatch.md index 370b21e0..f472c4a1 100644 --- a/.claude/rules/grammar-dispatch.md +++ b/.claude/rules/grammar-dispatch.md @@ -182,6 +182,19 @@ where the wrapper's span contains the leaf's, listing both is a no-op, because `LineSet::insert` / `insert_range` and `check_comment_ends_on_code_line` are all idempotent. +**Skip invented tokens by width, not by `is_missing()`.** Error recovery +inserts zero-width tokens for what the source lacks. It does so on +*valid* code the grammar cannot parse, as well as on invalid code. Some +grammars build these tokens without the MISSING flag: the C# and Elixir +zero-width identifiers are the known cases. So a guard meaning "this +node spells no written token" must test `start_byte() == end_byte()`, +and needs one malformed fixture per grammar it covers. + +Width is not a malformed-input test: valid trees have zero-width nodes +too, such as an empty Bash heredoc body. #1546 billed an operand spelled +`""` in all 23 languages, and its `is_missing()`-only fix still failed +C# and Elixir (lesson 95). + **Decide a grammar-reachable but language-invalid shape on what the change buys for *valid* input.** Grammars are routinely more permissive than their languages, so "the parser can produce this" does not settle diff --git a/.claude/rules/testing.md b/.claude/rules/testing.md index 0ede1d28..bb317cb9 100644 --- a/.claude/rules/testing.md +++ b/.claude/rules/testing.md @@ -118,6 +118,26 @@ equally common: two candidate rankings in #1465 were merely *different valid orderings* rather than defects, and the honest fix was to trim the test comment's claim. +**Re-run the perturbations when two predicates merge.** When two guards +are folded into one shared predicate, re-run the perturbation for every +test that guarded either one. A test can stop discriminating without +failing, because the merged paths can offset each other. During #1539, +once Perl ABC's `and` gate was shared, removing it left one slot row +reading 1: the slot dropped by one and the chain walk gained one. The +row's comment still claimed to guard `perl_condition_scores_itself`, +and only a re-run perturbation showed it no longer did. The comment was +corrected under #1539. + +**Run a perturbation sweep of an imported Python module with +`python3 -B`.** CPython reuses a cached `.pyc` when the source's mtime +(to the second) and size both match. A same-length edit such as `min` → +`max`, restored within the same second, therefore runs stale bytecode on +the next import. In #1562 that produced "isolated test fails, full run +passes" results that described the cache, not the code. Delete the +module's `__pycache__` before the sweep, then run every step with +`python3 -B` (or `PYTHONDONTWRITEBYTECODE=1`). `-B` stops new writes, but +a `.pyc` written before the sweep is still read. + After restoring, `git status` / `git diff --stat` must show exactly the edits you intend — nothing extra, nothing missing. diff --git a/AGENTS.md b/AGENTS.md index 4cedaa9f..ff2204ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -568,6 +568,23 @@ metric-value-only (no structural changes). Run `cargo insta test one at a time can shift `assertion_line` fields, causing a cascade where previously-matching snapshots become stale. +Three traps in that loop: + +- **The corpus runner aborts after its first panics**, so a plain run + reports only part of what moved. #1296 showed 16 of 254 files. + Collect with `INSTA_FORCE_PASS=1 cargo test --no-fail-fast` before + counting or reviewing. +- **A forced pass also hides inline snapshots.** An + `assert_json_snapshot!(…, @"…")` that moved passes and leaves only a + `src/**/.*.pending-snap` file, not a `.snap.new`. Review both kinds + (#1552). +- **`cargo insta test --accept` accepts every pending snapshot it finds + in the workspace**, not only the ones its own run produced. That + includes corpus snapshots left by an earlier forced run (#1486). The + per-test-file advice above holds only when nothing else is pending. + Otherwise review everything that is pending, or accept a filtered set + with `cargo insta accept --snapshot `. + **Anchor every `insta::assert_json_snapshot!` call.** A bare `insta::assert_json_snapshot!(metric.X)` records whatever production emitted at acceptance time — including bugs (see issue #95 and lesson 2 diff --git a/docs/development/lessons_learned.md b/docs/development/lessons_learned.md index 8eb89ff9..f8a3e1e1 100644 --- a/docs/development/lessons_learned.md +++ b/docs/development/lessons_learned.md @@ -138,6 +138,7 @@ number and the higher number stays as a redirect. | [92](#92-an-optimizations-rationale-can-encode-the-waste-it-optimizes-for) | An optimization's rationale can encode the waste it optimizes for | | [93](#93-a-gate-that-reads-a-typed-accessor-is-invisible-to-the-front-ends-that-read-the-wire) | A gate that reads a typed accessor is invisible to the front-ends that read the wire | | [94](#94-a-sweep-that-finds-nothing-only-rules-out-what-its-predicate-can-express) | A sweep that finds nothing only rules out what its predicate can express | +| [95](#95-error-recovery-inserts-tokens-that-is_missing-does-not-flag) | Skip invented tokens by width, not `is_missing()` — valid code triggers it too | --- @@ -1481,6 +1482,14 @@ from TS's `is_string` (and TSX's `String3`) rather than adding it to is not a literal in either predicate, which is the resolution to reach for when the cross-walk turns one of these up. +**A token arm narrowed without its slot predicate** (#1540). The first +draft of the fix narrowed Perl ABC's `=~` arm so that a bound +substitution (`$x =~ s///`) no longer scored as a test by use. Because +`perl_condition_scores_itself` still listed `=~` as scoring itself, the +slot paid nothing, and `if ($x =~ s///)` fell from 1 to 0. The fix that +shipped gates both on one shared predicate. A perturbation of each side +fails a different subset of tests. + --- ## 36. `serde_json::to_value` re-sorts JSON object keys via `BTreeMap` @@ -3881,3 +3890,40 @@ never whether it is compiled wherever a caller is, though the derivation already listed `go` among the helper's needs (#1528). --- + +## 95. Error recovery inserts tokens that `is_missing()` does not flag + +**Lesson:** A guard meant to skip what the parser invented rather than +what the author wrote must key on **width** (`start_byte() == +end_byte()`), not on `Node::is_missing()`. Prove it with one malformed +fixture per grammar the guard covers. Do not assume the trap needs +invalid input: grammars cannot parse every valid program, and they +recover from the gap the same way. + +When tree-sitter recovers from a parse error, it can insert a token the +source lacks, such as a `;`, a `)` or an identifier. The inserted token +has zero width. Most are flagged MISSING. Some grammars' recovery also +builds zero-width identifiers *without* that flag. A classifier treats +either kind as an ordinary leaf, so the result is a plausible count, +not an error. Fixtures written from code the grammar handles never +contain one. + +**An operand spelled `""` in every language** (#1546). + +- Every getter that classified identifiers billed an inserted identifier + as an empty-text operand. An inserted `;` or `)` became a second + operator. +- Most instances were *valid* C++ that tree-sitter-cpp 0.23.4 cannot + parse, such as a `typename` functional cast used as a statement, or the + templated destructor call `dfs_state->~DfsState()`. There were + roughly 9,900 inserted `;` across DeepSpeech. +- The obvious fix, skipping `is_missing()` nodes, cleared C++ but left C# + and Elixir billing `""`, because their recovery builds the identifier + unflagged. A sweep with one malformed fixture per grammar showed it. + `compute_halstead` now skips any zero-width node, since a zero-width + node spells no token. +- Width answers "was a token written", not "is the input malformed": + valid trees contain zero-width nodes too, such as an empty Bash heredoc + body. + +--- From fe58a7151703da1199dc9a5b1e93b81d59869f4c Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:34:13 -0700 Subject: [PATCH 42/48] fix: address the whole-branch review findings Two reviews of main...fix/batch-2026-10-09 found defects that the per-issue pipelines could not see. Each behavioural fix carries a regression test, verified by reverting the fix. - Perl ABC and Bash cognitive/ABC walks were quadratic in chain length or statement count; climb only where the slot pays, and track Bash comparison runs top-down. - Kotlin soft keywords used as names billed as modifiers (#1558). - C++ operator() / operator[] rendered as () / [], duplicating bca ops entries (#1296). - Generic <> gated against comparisons the ObjC and C# grammars misparse as generic lists (#1559). - check-test-lang-gates: three wrong-verdict paths for guarded calls and const tables (#1562). - Perl and / -and / -not keys, a signed key, and infix minus (#1539, #1541, #1545). - Elixir Kernel.|| keeps its cyclomatic decision; Foo.|| does not (#1534). - TS export { type }, C++ co_return ABC, Bash =~ sequences. - CHANGELOG and book corrections, including the halstead.effort default drift tracked in #1569. --- CHANGELOG.md | 62 +++- big-code-analysis-ast/src/getter.rs | 5 +- big-code-analysis-ast/src/getter/cpp.rs | 25 +- big-code-analysis-ast/src/getter/csharp.rs | 34 ++ .../src/getter/generic_angles.rs | 20 +- big-code-analysis-ast/src/getter/groovy.rs | 7 +- big-code-analysis-ast/src/getter/java.rs | 1 + big-code-analysis-ast/src/getter/js_family.rs | 31 ++ big-code-analysis-ast/src/getter/kotlin.rs | 131 ++++++-- big-code-analysis-ast/src/getter/mozcpp.rs | 15 +- big-code-analysis-ast/src/getter/objc.rs | 23 +- big-code-analysis-ast/src/getter/perl.rs | 21 +- big-code-analysis-ast/src/getter/rust.rs | 6 +- big-code-analysis-ast/src/getter/tsx.rs | 5 +- .../src/getter/typescript.rs | 10 +- .../src/lang_helpers/bash.rs | 50 +-- .../src/lang_helpers/elixir.rs | 47 ++- .../src/lang_helpers/perl.rs | 67 +++- big-code-analysis-book/src/metrics.md | 33 +- src/metrics/abc.rs | 45 +++ src/metrics/abc/bash.rs | 9 +- src/metrics/abc/cpp.rs | 6 +- src/metrics/abc/mozcpp.rs | 4 +- src/metrics/abc/perl.rs | 132 +++++--- src/metrics/bash_test_connective_tests.rs | 62 +++- src/metrics/cognitive.rs | 2 + src/metrics/cognitive/bash.rs | 96 ++++-- src/metrics/cyclomatic/elixir.rs | 15 +- src/metrics/elixir_named_operator_tests.rs | 39 ++- src/metrics/halstead.rs | 312 +++++++++++++++--- src/metrics/mod.rs | 6 +- src/metrics/perl_keyword_key_tests.rs | 117 ++++++- src/ops.rs | 102 +++++- src/spaces_tests.rs | 27 +- tests/api/ast_seam_test.rs | 4 - utils/check-test-lang-gates-test.py | 143 ++++++++ utils/check-test-lang-gates.py | 101 +++++- 37 files changed, 1522 insertions(+), 293 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2770f260..faeebc38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,14 +41,21 @@ move for most Python, Rust, C, C++, Java, JavaScript, TypeScript, Kotlin, Groovy and C# code. Python and Rust rise most, from #1486. The generic `<>` pair (#1559) lowers effort for most code that uses generics. Several **Fixed** entries also move only Halstead and are -marked as such. +marked as such. If you gate `halstead.effort` or `mi.*` with `bca +check`, expect new offenders in Python, Rust, Java, C# and C++ code, +and refresh `.bca-baseline.toml`. The `bca init` default for +`halstead.effort` and the book's per-language calibration table were +derived before these changes. Their re-derivation is tracked in #1569. ### Added - **`Getter::get_operator_spelling`** (#1559), a defaulted trait method - that lets a getter bill an operator under a fixed text key when its - grammar gives two operators the same kind id. It is additive: existing - implementations compile unchanged. + in `big-code-analysis-ast` that lets a getter bill an operator under a + fixed text key when its grammar gives two operators the same kind id. + It is additive: existing implementations compile unchanged. Like the + rest of `big-code-analysis-ast`, it is outside the stability contract + (see `STABILITY.md`). Nothing new reaches the `big-code-analysis` + crate's public surface. ### Changed @@ -72,14 +79,18 @@ marked as such. `namespace`, `module`, `declare` and `global`. Every TypeScript `type_identifier`, including alias and interface names, annotations and generic arguments, is now an operand. `template ` and - `template ` bill alike. + `template ` bill alike. A binding named `type` in an export + clause (`export { type }`) is not billed as the keyword. - **Java type names are Halstead operands** (#1560), as in C#, Rust and TypeScript. A class literal's interior and the inferred `var` stay unbilled. - **Kotlin bills its declaration modifiers and `interface`** (#1556, #1558): `enum`, `data`, `sealed`, the visibility keywords, `open`, `override`, `suspend`, `reified`, variance `out` and the rest. Before - this, only `class` and `object` were billed. + this, only `class` and `object` were billed. The same words used as + names bill no operator, including where the grammar fails to parse + the name (`open = !open`, `val suspend = 1`, a parameter named + `vararg`). - **TypeScript bills `satisfies`, `keyof`, `infer`, `is` and `asserts`** (#1561). - **Generic angle brackets bill as one `<>` pair** (#1559) in Rust (type @@ -87,11 +98,16 @@ marked as such. Groovy, C#, TypeScript, TSX, Kotlin, C++ templates and Objective-C (lightweight generics, protocol lists and `id

`). The opening `<` bills `<>` and the closing `>` bills nothing, the same convention as - `()`, `[]` and `{}`. Comparisons and shifts are unchanged. N1 falls + `()`, `[]` and `{}`. Comparisons and shifts are unchanged, with two + exceptions where the grammar reads a comparison as a generic list. + The first is a C++ comparison that tree-sitter-cpp reads as a template + argument list (`x = a < b || c > (d)`). The second is a C# call + argument pair spelled with `>=` (`F(a < b, c >= d)`), which the + grammar parses as an assignment. Both bill as `<>`. N1 falls for code that uses generics, and effort fell in about 87% of the moved C++ and Rust corpus files. - **Perl and PHP `xor` scores as a value, not a decision** (#1536), like - `^` and the eager `xor` of Java, C++ and Kotlin. It adds no + Java's `^` and the eager `xor` of C++ and Kotlin. It adds no cyclomatic or cognitive increment, and the boolean slot holding it pays one ABC condition. PHP `xor` moves in all three metrics. - **Rust `let`-`else` is a decision** (#1542, #1548). It adds one @@ -291,7 +307,12 @@ marked as such. - A `-bareword` key (`$h{-foo}`, `(-text => 1)`) was read as a file test, which billed a fragment of the word (`oo`) and an ABC branch. Each key now scores like its quoted or `or` twin, and real operators - and file tests are unchanged. + and file tests are unchanged. ABC still reads the value after an + `and`, `not`, `-and` or `-not` key, so `f(and => !$a)` pays the + negation's condition. A `-and` / `-not` key keeps its sign however it + is spaced (`(- not => 1)`), and a real subtraction before a key + (`(1 - and => 2)`) keeps its `-`. A `-bareword` key whose letter is + no file-test letter (`-name`) still scores as a negated bareword. - **Perl `$x =~ s///` and `tr///` no longer score a condition by use** (#1540), matching bare `s///`. A substitution is an edit, not a test. Inside a boolean slot it still counts once, and pattern matches are @@ -304,10 +325,15 @@ marked as such. (#1536). `[ X -a Y ]`, `[ X ] && [ Y ]` and `[[ X && Y ]]` now agree in cyclomatic, ABC and cognitive. `-a` / `-o` are logical connectives, not ABC comparisons, and `=` inside a test is an ABC comparison. It - stays an assignment in `(( … ))`. + stays an assignment in `(( … ))`. A run of `=` or `=~` tests joined by + one connective, which the grammar nests around the connectives, is one + cognitive sequence, like its `==` twin. - **An Elixir named operator is billed once** (#1534). `&==/2`, `&and/2` and `Kernel.||(a, b)` billed both an operand and an operator, - and a named `&&` / `||` / `and` / `or` added a cyclomatic decision. + and a captured `&&` / `||` / `and` / `or` (`&||/2`) added a cyclomatic + decision. A remote call on `Kernel` (`Kernel.||(a, b)`) applies the + operator and keeps its decision. The same call on a user module + (`Foo.||(a, b)`) is an ordinary function call and decides nothing. - **C# lambda modifiers and implicit parameters are billed** (#1482, #1544). A lambda's or anonymous method's `static` / `async` is an operator, keyed as the declaration spelling is. The bare parameter of @@ -316,9 +342,12 @@ marked as such. covers C# `operator true` / `operator false` (previously operands), and C++ / Mozcpp `operator[]`, `operator()`, `operator""_x`, `operator co_await` and the `operator` keyword (previously billed - nowhere). `co_await`, `co_return` and `co_yield` in an expression are - operators too, and NExits counts `co_return` and `co_yield` as exits - (#1547). + nowhere); `bca ops` lists the C++ names as `operator[]` and + `operator()`, apart from a subscript's `[]` and a call's `()`. So are + `co_await` in an expression and the `co_return` and `co_yield` + statements, which NExits now counts as exits (#1547). ABC reads a + `co_return` value as it reads a `return` value, so `co_return !x` pays + one condition. - **JSX element text is a Halstead operand** (#1483), keyed on its trimmed text, in JavaScript, Mozjs and TSX. Whitespace-only text is ignored, and a character reference such as `&` in element content @@ -337,7 +366,10 @@ marked as such. lowers `cyclomatic.sum` by one base per removed space. - **`make check-test-lang-gates` catches a helper gated out of a build its caller compiles in** (#1562), when the caller names its parsers - only inside an inner `#[cfg]` block. + only inside an inner `#[cfg]` block. A `const` fixture table passed + only to such calls is checked the same way. +- **Perl's fat comma lists as `=>` in `bca ops`**, not as the grammar's + `fat_comma` node name. Rendering only; no count moves. ## [2.3.0] - 2026-10-07 diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index f9b53a9e..85278aae 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -17,7 +17,7 @@ use crate::lang_helpers::tcl_family::{ }; use crate::space_kind::SpaceKind; use crate::traits::Search; -use generic_angles::GenericAngleKinds; +use generic_angles::{GenericAngleKinds, never_misparsed}; use js_family::impl_js_family_get_op_type; use crate::*; @@ -322,7 +322,8 @@ pub trait Getter { /// different operator; `None` keys it by kind. A generic list's `<` /// is the case (#1559): it has the comparison's kind but is keyed as /// the `"<>"` pair, as `()`, `[]` and `{}` are. Consulted only for a - /// node [`get_op_type`](Self::get_op_type) calls an operator. + /// node [`get_op_type_with_code`](Self::get_op_type_with_code) calls + /// an operator. #[inline] #[must_use] fn get_operator_spelling<'a>( diff --git a/big-code-analysis-ast/src/getter/cpp.rs b/big-code-analysis-ast/src/getter/cpp.rs index b3d3f983..89ac98ee 100644 --- a/big-code-analysis-ast/src/getter/cpp.rs +++ b/big-code-analysis-ast/src/getter/cpp.rs @@ -14,6 +14,7 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { ], openers: &[Cpp::LT as u16], closers: &[Cpp::GT as u16, Cpp::GT2 as u16], + is_misparse: never_misparsed, }; impl Getter for CppCode { @@ -173,9 +174,9 @@ impl Getter for CppCode { // // `LPARENRPAREN` / `LBRACKRBRACK` are kinds of their own, not // the `LPAREN` / `LBRACK` of a call or subscript, so declaring - // `operator[]` and applying `s[1]` are two `n1` entries that - // `bca ops` both renders `[]` — the same choice Ruby makes for - // its `def [](i)` method names. + // `operator[]` and applying `s[1]` are two `n1` entries, which + // `bca ops` renders `operator[]` and `[]` (see + // `get_operator_id_as_str`). | LPARENRPAREN | LBRACKRBRACK | DQUOTEDQUOTE | CoAwait | Operator // `co_return` and `co_yield` are the coroutine spellings of // `return` and of handing a value back, so they bill as @@ -285,5 +286,21 @@ impl Getter for CppCode { GENERIC_ANGLES.opener_spelling(node, ancestors) } - get_operator!(Cpp); + // `get_operator!`'s three bracket arms, plus the two overloaded + // operator names (#1296). The grammar names `LPARENRPAREN` / + // `LBRACKRBRACK` `()` / `[]`, the same text a call's and a + // subscript's opener render as, so `bca ops` would list one spelling + // for two `n1` entries. Rendering each as the operator it names keeps + // every listed operator distinct (lesson 4). + fn get_operator_id_as_str(id: u16) -> &'static str { + let typ = id.into(); + match typ { + Cpp::LPAREN => "()", + Cpp::LBRACK => "[]", + Cpp::LBRACE => "{}", + Cpp::LPARENRPAREN => "operator()", + Cpp::LBRACKRBRACK => "operator[]", + _ => typ.into(), + } + } } diff --git a/big-code-analysis-ast/src/getter/csharp.rs b/big-code-analysis-ast/src/getter/csharp.rs index 84a24f9b..00024916 100644 --- a/big-code-analysis-ast/src/getter/csharp.rs +++ b/big-code-analysis-ast/src/getter/csharp.rs @@ -10,8 +10,42 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { ], openers: &[Csharp::LT as u16], closers: &[Csharp::GT as u16], + is_misparse: csharp_comparison_misparse, }; +// Whether `list` holds the two comparisons of a call argument +// (`F(a < b, c > d)`) that tree-sitter-c-sharp reads as a declaration +// typed `a` and named `d`. The C# specification keeps the type +// arguments only where the token after `>` allows it (§6.2.5), and an +// identifier allows it only after `is`, `case`, `out` or in a tuple — +// so a declaration typed by a generic name is two comparisons when it is +// a call or indexer argument without `out`. `F(out List x)`, +// `(List p, int q) = …` and `o is List x` keep their pair. A +// member access before the `<` (`F(x.a < b, c > d)`) wraps the generic +// name in `qualified_name`s, which the climb passes. The `>=` spelling +// (`F(a < b, c >= d)`) parses as an assignment instead and is not undone. +fn csharp_comparison_misparse<'a>(list: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let mut up = ancestors.iter(list).map(|(node, _)| node); + if up + .next() + .is_none_or(|name| name.kind_id() != Csharp::GenericName as u16) + { + return false; + } + let mut up = up.skip_while(|node| node.kind_id() == Csharp::QualifiedName as u16); + let (Some(declaration), Some(argument), Some(arguments)) = (up.next(), up.next(), up.next()) + else { + return false; + }; + declaration.kind_id() == Csharp::DeclarationExpression as u16 + && argument.kind_id() == Csharp::Argument as u16 + && !argument.is_child(Csharp::Out as u16) + && matches!( + arguments.kind_id().into(), + Csharp::ArgumentList | Csharp::BracketedArgumentList + ) +} + impl Getter for CsharpCode { fn get_space_kind(node: &Node) -> SpaceKind { use Csharp::*; diff --git a/big-code-analysis-ast/src/getter/generic_angles.rs b/big-code-analysis-ast/src/getter/generic_angles.rs index cb0fc2c6..ff67d09a 100644 --- a/big-code-analysis-ast/src/getter/generic_angles.rs +++ b/big-code-analysis-ast/src/getter/generic_angles.rs @@ -27,13 +27,27 @@ pub(crate) struct GenericAngleKinds { pub(crate) openers: &'static [u16], /// The `>` kinds, likewise. pub(crate) closers: &'static [u16], + /// Whether a list node, given its ancestor chain, holds comparisons + /// the grammar misparsed as a generic list, whose brackets then keep + /// billing as the comparisons they are. [`never_misparsed`] for a + /// grammar with no such shape this can tell from a real list. + pub(crate) is_misparse: for<'a, 'b> fn(&Node<'a>, Ancestors<'a, 'b>) -> bool, +} + +/// The [`GenericAngleKinds::is_misparse`] of a grammar whose generic +/// lists hold no misparse this can tell apart. C++ is one: tree-sitter-cpp +/// reads `a < b || c > (d)` as a template argument list, but so does a +/// valid template call spelled the same, and only name lookup tells them +/// apart. +pub(crate) fn never_misparsed(_: &Node, _: Ancestors) -> bool { + false } impl GenericAngleKinds { fn parent_is_list<'a>(&self, node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - ancestors - .parent(node) - .is_some_and(|parent| self.lists.contains(&parent.kind_id())) + ancestors.iter(node).next().is_some_and(|(list, above)| { + self.lists.contains(&list.kind_id()) && !(self.is_misparse)(&list, above) + }) } /// Whether `node` closes a generic list, and so bills nothing. diff --git a/big-code-analysis-ast/src/getter/groovy.rs b/big-code-analysis-ast/src/getter/groovy.rs index a268f6c7..a93d2a35 100644 --- a/big-code-analysis-ast/src/getter/groovy.rs +++ b/big-code-analysis-ast/src/getter/groovy.rs @@ -6,7 +6,9 @@ use super::*; // `method_type_parameters` is the method-level spelling of // `type_parameters` (` T id(T x)`). `LT2` is the `token.immediate` // `<` that `type_parameters` and `type_arguments` open with; the parser -// reports it as `LT`, so it is listed defensively. +// reports it as `LT`, so it is listed defensively, here and in the +// operator arm, and its absence is pinned by +// `groovy_generic_opener_alias_never_reaches_kind_id`. const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { lists: &[ Groovy::TypeArguments as u16, @@ -15,6 +17,7 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { ], openers: &[Groovy::LT as u16, Groovy::LT2 as u16], closers: &[Groovy::GT as u16], + is_misparse: never_misparsed, }; impl Getter for GroovyCode { @@ -203,7 +206,7 @@ impl Getter for GroovyCode { // Separators / brackets. | SEMI | COMMA | COLONCOLON | DOT | DASHGT | LBRACE | LBRACK | LPAREN // Java-compatible operators (arithmetic, bitwise, comparison, assignment). - | EQ | LT | GT | BANG | TILDE | QMARK | COLON | EQEQ | LTEQ | GTEQ | BANGEQ + | EQ | LT | LT2 | GT | BANG | TILDE | QMARK | COLON | EQEQ | LTEQ | GTEQ | BANGEQ | AMPAMP | PIPEPIPE | PLUSPLUS | DASHDASH | PLUS | DASH | STAR | SLASH | AMP | PIPE | CARET | PERCENT | LTLT | GTGT | GTGTGT | PLUSEQ | DASHEQ | STAREQ | SLASHEQ | AMPEQ | PIPEEQ | CARETEQ | PERCENTEQ | LTLTEQ | GTGTEQ | GTGTGTEQ diff --git a/big-code-analysis-ast/src/getter/java.rs b/big-code-analysis-ast/src/getter/java.rs index 6dc818c7..b5439162 100644 --- a/big-code-analysis-ast/src/getter/java.rs +++ b/big-code-analysis-ast/src/getter/java.rs @@ -7,6 +7,7 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { lists: &[Java::TypeArguments as u16, Java::TypeParameters as u16], openers: &[Java::LT as u16], closers: &[Java::GT as u16], + is_misparse: never_misparsed, }; impl Getter for JavaCode { diff --git a/big-code-analysis-ast/src/getter/js_family.rs b/big-code-analysis-ast/src/getter/js_family.rs index 32da5887..dc9aea02 100644 --- a/big-code-analysis-ast/src/getter/js_family.rs +++ b/big-code-analysis-ast/src/getter/js_family.rs @@ -79,6 +79,11 @@ // trimmed. A token keeps the newlines and indentation around it, so // `Start` at two indentation depths would otherwise be two distinct // operands. This is the same shape as Kotlin's #454 narrowing. +// Only the ends are trimmed: a multi-line text keeps its inner +// newlines and indentation, so the same lines at two depths, or +// wrapped differently, stay distinct operands although JSX joins +// them into one string. Folding them needs an owned key, and +// `get_operand_id` returns a slice of the source. // // An `html_character_reference` (`&`) splits the text around it // into siblings, so `a & b` is three operands: `a`, `&` and @@ -112,6 +117,7 @@ macro_rules! impl_js_family_get_op_type { operand_extras: [$($operand_extra:ident),* $(,)?] $(, predefined_void: $predefined_type:ident)? $(, generic_angles: $generic_angles:ident)? + $(, export_type_name: [$type_keyword:ident, $export_clause:ident, $export_specifier:ident])? $(, jsx: [$jsx_text:ident, $jsx_entity:ident])? $(,)? ) => { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { @@ -123,6 +129,31 @@ macro_rules! impl_js_family_get_op_type { } )? + // TS/TSX only: a binding *named* `type` in an export clause + // (`export { type }`, `export { type as kind }`). The grammar's + // `export_specifier` has no slot for that name, unlike + // `import_specifier`, so it recovers the keyword leaf under an + // `ERROR` in the clause, or beside one in the specifier, and + // the `type` operator would bill a binding name. A type-only + // specifier (`export { type Foo }`) parses cleanly and still + // bills, as does the `type` of `export type * from "m"`, whose + // `ERROR` hangs from the statement rather than the clause. + $( + if node.kind_id() == $type_keyword as u16 && { + let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); + up.next().is_some_and(|parent| { + if parent.is_error() { + up.next() + .is_some_and(|clause| clause.kind_id() == $export_clause as u16) + } else { + parent.kind_id() == $export_specifier as u16 && parent.has_error() + } + }) + } { + return TokenRole::Unknown; + } + )? + // TS/TSX only: a `void` return / parameter type is parsed as a // `predefined_type` wrapper around an inner `void` token. Both // the wrapper (routed through `is_primitive` into the text-keyed diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index 762130a8..e2f4e9e2 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -122,12 +122,57 @@ fn kotlin_string_has_interp(node: &Node, code: &[u8]) -> bool { }) } +// Whether `node`, a modifier keyword leaf, sits in a modifier list the +// grammar built (#1558), rather than in the recovery of a soft-keyword +// *name* it failed to parse. Two recoveries strand the leaf: +// +// - an `ERROR` holds it, usually still wrapped in its `*_modifier` +// node (`open = !open` at statement start, `val suspend = 1`), so the +// first ancestor above the wrappers is the `ERROR`; +// - a modifier list is followed by an inserted, zero-width name +// (`fun f(vararg: Int)`, `class C(open: Int)`). +// +// Kinds are compared by name: `*_modifier` covers every wrapper, and +// `user_type` is where recovery can hang `type_modifiers`. The inserted +// name is found by width, not `is_missing()` (grammar-dispatch). +fn kotlin_modifier_was_written_as_one<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let host = ancestors + .iter(node) + .map(|(ancestor, _)| ancestor) + .find(|ancestor| { + let kind = ancestor.kind(); + !(kind.ends_with("_modifier") || kind == "type_modifiers" || kind == "user_type") + }); + host.is_some_and(|host| !host.is_error()) && !kotlin_next_token_is_inserted(node, ancestors) +} + +// Whether the first token after `node` is zero-width: one error recovery +// inserted where the source has none. +fn kotlin_next_token_is_inserted<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let mut current = *node; + for (parent, _) in ancestors.iter(node) { + if let Some(mut next) = parent + .children() + .skip_while(|child| child.id() != current.id()) + .nth(1) + { + while let Some(first) = next.child(0) { + next = first; + } + return next.start_byte() == next.end_byte(); + } + current = parent; + } + false +} + // `super` (a `super_expression`) names a supertype rather than // listing type arguments, so its brackets stay `<` and `>`. const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { lists: &[Kotlin::TypeArguments as u16, Kotlin::TypeParameters as u16], openers: &[Kotlin::LT as u16], closers: &[Kotlin::GT as u16], + is_misparse: never_misparsed, }; impl Getter for KotlinCode { @@ -225,33 +270,77 @@ impl Getter for KotlinCode { return TokenRole::Unknown; } - match node.kind_id().into() { + // Modifier keywords, billed as C# bills all of its own (#1558; + // Java and Groovy bill only some, #1565). Each is the keyword + // leaf; its `*_modifier` / `modifiers` wrappers stay unlisted, so + // one modifier bills once (grammar-dispatch section 5). Every + // keyword has exactly one kind, so `kind_id` keys it without + // splitting it across the two `n1` maps. `Annotation2` is the + // `annotation` of `annotation class`; `Annotation` is an `@N` use, + // which is no keyword. `in` already bills as a variance modifier + // through `In` below, so `out` joins it. `reified` has only the + // `ReificationModifier` kind, which `KotlinCode::is_primitive` + // keys by its text. An early return rather than an arm, because + // the match below is one rustfmt declines to format. + // + // Each word is a soft keyword that valid Kotlin may use as a name, + // and where kotlin-ng fails to parse that name the leaf survives + // in its recovery (`open = !open` at statement start, + // `val suspend = 1`, a parameter named `vararg`). Such a leaf is + // no modifier and bills nothing; see + // `kotlin_modifier_was_written_as_one`. `companion` is left out + // of that gate: it never parses as a name, and gating it would + // only drop a real `companion object` from a class body that + // failed to parse. + let kind: Kotlin = node.kind_id().into(); + if matches!( + kind, + Enum | Sealed + | Annotation2 + | Data + | Inner + | Value + | Public + | Private + | Protected + | Internal + | Abstract + | Final + | Open + | Override + | Lateinit + | Const + | Tailrec + | Kotlin::Operator + | Infix + | Inline + | External + | Suspend + | Vararg + | Noinline + | Crossinline + | ReificationModifier + | Out + | Expect + | Actual + ) { + return if kotlin_modifier_was_written_as_one(node, ancestors) { + TokenRole::Operator + } else { + TokenRole::Unknown + }; + } + + match kind { // Operator: control flow keywords If | Else | When | For | While | Do | Try | Catch | Finally | Throw | Return | ReturnAT // Operator: other keywords. `interface` is the declaration // keyword leaf beside `class` and `object` (#1556); its - // `class_declaration` wrapper stays unlisted. + // `class_declaration` wrapper stays unlisted. `companion` is + // the one modifier billed here; see the early return above. | Class | Interface | Fun | Object | Val | Var | In | Is | As | AsQMARK | BANGis - | BANGin | Constructor - // Operator: modifier keywords, as C# bills all of its own - // (#1558; Java and Groovy bill only some, #1565). Each is - // the keyword leaf; its `*_modifier` / `modifiers` wrappers - // stay unlisted, so one modifier bills once - // (grammar-dispatch section 5). Every - // keyword has exactly one kind, so `kind_id` keys it without - // splitting it across the two `n1` maps. `Annotation2` is the - // `annotation` of `annotation class`; `Annotation` is an `@N` - // use, which is no keyword. `in` already bills as a variance - // modifier through `In` above, so `out` joins it. `reified` - // has only the `ReificationModifier` kind, which - // `KotlinCode::is_primitive` keys by its text. - | Enum | Sealed | Annotation2 | Data | Inner | Value | Companion - | Public | Private | Protected | Internal - | Abstract | Final | Open | Override | Lateinit | Const - | Tailrec | Kotlin::Operator | Infix | Inline | External | Suspend - | Vararg | Noinline | Crossinline | ReificationModifier | Out - | Expect | Actual + | BANGin | Constructor | Companion // Operator: brackets, separators, terminators | SEMI | COMMA | COLONCOLON | DOT | LBRACE | LBRACK | LPAREN // Operator: assignment and arithmetic diff --git a/big-code-analysis-ast/src/getter/mozcpp.rs b/big-code-analysis-ast/src/getter/mozcpp.rs index e2d8435d..c822ec32 100644 --- a/big-code-analysis-ast/src/getter/mozcpp.rs +++ b/big-code-analysis-ast/src/getter/mozcpp.rs @@ -12,6 +12,7 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { ], openers: &[Mozcpp::LT as u16], closers: &[Mozcpp::GT as u16, Mozcpp::GT2 as u16], + is_misparse: never_misparsed, }; impl Getter for MozcppCode { @@ -175,5 +176,17 @@ impl Getter for MozcppCode { GENERIC_ANGLES.opener_spelling(node, ancestors) } - get_operator!(Mozcpp); + // The overloaded operator names render as the operator they name; + // the derivation is on the Cpp twin (#1296). + fn get_operator_id_as_str(id: u16) -> &'static str { + let typ = id.into(); + match typ { + Mozcpp::LPAREN => "()", + Mozcpp::LBRACK => "[]", + Mozcpp::LBRACE => "{}", + Mozcpp::LPARENRPAREN => "operator()", + Mozcpp::LBRACKRBRACK => "operator[]", + _ => typ.into(), + } + } } diff --git a/big-code-analysis-ast/src/getter/objc.rs b/big-code-analysis-ast/src/getter/objc.rs index a5ff376b..c94315d4 100644 --- a/big-code-analysis-ast/src/getter/objc.rs +++ b/big-code-analysis-ast/src/getter/objc.rs @@ -9,11 +9,13 @@ use crate::c_declarator::declarator_name; // `protocol_reference_list` (`id`), `parameterized_arguments` // (`@interface A : NSObject `), and `argument_list`, whose // `Type<…>` alternative (`f(NSArray)`) holds the brackets as -// direct children. A comparison's `<` / `>` is always the child of a -// `binary_expression` or `preproc_binary_expression`, so none of the -// four can claim one. `LT2` is the +// direct children. A well-formed comparison's `<` / `>` is the child of +// a `binary_expression` or `preproc_binary_expression`; the one +// exception is `objc_argument_list_misparse`. `LT2` is the // `token.immediate` `<` that `argument_list`'s form opens with; the -// parser reports it as `LT`, so it is listed defensively. The call's +// parser reports it as `LT`, so it is listed defensively, here and in +// the operator arm, and its absence is pinned by +// `objc_generic_opener_alias_never_reaches_kind_id`. The call's // `argument_list` is `ArgumentList2`; `ArgumentList` is // `preproc_call_expression`'s alias, which holds no angle brackets. const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { @@ -25,8 +27,19 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { ], openers: &[Objc::LT as u16, Objc::LT2 as u16], closers: &[Objc::GT as u16], + is_misparse: objc_argument_list_misparse, }; +// Whether `list` is an `argument_list` holding two comparisons that +// tree-sitter-objc read as its `Type<…>` form: an unspaced +// `g(ad)`, where a `<` glued to the first argument opens the form. +// The real form is the whole argument list, so `f(NSArray)` +// parses cleanly, while the misparse leaves an `ERROR` after the `>`, or +// a MISSING `)` (`g(i=0)`) that only `has_error` sees. +fn objc_argument_list_misparse(list: &Node, _ancestors: Ancestors) -> bool { + list.kind_id() == Objc::ArgumentList2 as u16 && list.has_error() +} + impl Getter for ObjcCode { fn get_func_space_name<'a, 'tree>( node: &Node<'tree>, @@ -134,7 +147,7 @@ impl Getter for ObjcCode { DOT | LPAREN | LPAREN2 | COMMA | STAR | GTGT | COLON | SEMI | Return | Break | Continue | If | Else | Switch | Case | Default | For | While | Goto | Do | EQ | AMPAMP | PIPEPIPE | DASH | DASHDASH | DASHGT | PLUS | PLUSPLUS | SLASH | PERCENT - | PIPE | AMP | LTLT | TILDE | LT | LTEQ | EQEQ | BANGEQ | GTEQ | GT | PLUSEQ + | PIPE | AMP | LTLT | TILDE | LT | LT2 | LTEQ | EQEQ | BANGEQ | GTEQ | GT | PLUSEQ | DASHEQ | BANG | STAREQ | SLASHEQ | PERCENTEQ | GTGTEQ | LTLTEQ | AMPEQ | CARET | CARETEQ | PIPEEQ | LBRACK | LBRACE | QMARK | PrimitiveType | TypeSpecifier | Sizeof | Signed | Unsigned | Long | Short | In | AT | ATtry | ATcatch | ATfinally diff --git a/big-code-analysis-ast/src/getter/perl.rs b/big-code-analysis-ast/src/getter/perl.rs index c3a99095..12228f27 100644 --- a/big-code-analysis-ast/src/getter/perl.rs +++ b/big-code-analysis-ast/src/getter/perl.rs @@ -4,7 +4,7 @@ use super::*; use crate::lang_helpers::perl::{ perl_and_is_operator, perl_dash_signs_keyword_key, perl_file_test_is_dash_key, - perl_is_dash_key_word, perl_not_is_key, + perl_is_dash_key_word, perl_key_is_signed, perl_not_is_key, }; impl Getter for PerlCode { @@ -254,12 +254,14 @@ impl Getter for PerlCode { /// file test (`(-x => 1)`) has swallowed the `=>` and the value into /// its node, so the key is the sign and the word after it; a /// `-and` / `-not` key gets back the sign its `-` token stopped - /// billing. Only a key reaches here as an operand, so the byte - /// before `and` / `not` is that sign whenever it is a `-`. + /// billing, spelled the same however the two are spaced, since Perl + /// keys `(- not => 1)` as `"-not"` too. The sign is read off the tree + /// (`perl_key_is_signed`), not the byte before the key, which can + /// equally be an infix `-` or the end of a comment. fn get_operand_id<'a>( node: &Node<'a>, code: &'a [u8], - _ancestors: Ancestors<'a, '_>, + ancestors: Ancestors<'a, '_>, ) -> &'a [u8] { let (start, end) = (node.start_byte(), node.end_byte()); match node.kind_id().into() { @@ -270,11 +272,8 @@ impl Getter for PerlCode { .count(); &code[start..=start + word] } - Perl::And | Perl::Not - if start.checked_sub(1).is_some_and(|sign| code[sign] == b'-') => - { - &code[start - 1..end] - } + Perl::And if perl_key_is_signed(node, ancestors) => b"-and", + Perl::Not if perl_key_is_signed(node, ancestors) => b"-not", _ => &code[start..end], } } @@ -289,7 +288,8 @@ impl Getter for PerlCode { /// `substitution_pattern_s` and `transliteration_tr_or_y` among /// the operators, which reads as a bug rather than as `s///`. /// `tr///` covers `y///` too — one kind, one glyph, because they - /// are synonyms. + /// are synonyms. `=>` is the named `fat_comma` node for the same + /// reason. #[inline] fn get_operator_id_as_str(id: u16) -> &'static str { let typ = id.into(); @@ -299,6 +299,7 @@ impl Getter for PerlCode { Perl::LBRACE => "{}", Perl::SubstitutionPatternS => "s///", Perl::TransliterationTrOrY => "tr///", + Perl::FatComma => "=>", _ => typ.into(), } } diff --git a/big-code-analysis-ast/src/getter/rust.rs b/big-code-analysis-ast/src/getter/rust.rs index 8a0bb097..be49e11f 100644 --- a/big-code-analysis-ast/src/getter/rust.rs +++ b/big-code-analysis-ast/src/getter/rust.rs @@ -7,7 +7,8 @@ use super::*; // (`impl Tr + use<'a, T>`) are generic-parameter lists under other // names. `LT2` is the `token(prec(1, '<'))` spelling `type_arguments` // and `use_bounds` open with; the parser reports it as `LT`, so it is -// listed defensively. +// listed defensively, here and in the operator arm, and its absence is +// pinned by `rust_generic_opener_alias_never_reaches_kind_id`. const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { lists: &[ Rust::TypeArguments as u16, @@ -17,6 +18,7 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { ], openers: &[Rust::LT as u16, Rust::LT2 as u16], closers: &[Rust::GT as u16], + is_misparse: never_misparsed, }; impl Getter for RustCode { @@ -88,7 +90,7 @@ impl Getter for RustCode { // statement-heavy code. LPAREN | LBRACE | LBRACK | As | EQGT | PLUS | STAR | Async | Await | Break | Continue | Else | For | If | In | Let | Loop | Match | Return | Unsafe | While - | EQ | COMMA | DASHGT | QMARK | LT | GT | AMP | MutableSpecifier | DOTDOT + | EQ | COMMA | DASHGT | QMARK | LT | LT2 | GT | AMP | MutableSpecifier | DOTDOT | DOTDOTEQ | DASH | AMPAMP | PIPE | CARET | EQEQ | BANGEQ | LTEQ | GTEQ | LTLT | GTGT | PERCENT | PLUSEQ | DASHEQ | STAREQ | SLASHEQ | PERCENTEQ | AMPEQ | PIPEEQ | CARETEQ | LTLTEQ | GTGTEQ | Move | DOT | PrimitiveType | PrimitiveType2 diff --git a/big-code-analysis-ast/src/getter/tsx.rs b/big-code-analysis-ast/src/getter/tsx.rs index 7db304df..d71365b8 100644 --- a/big-code-analysis-ast/src/getter/tsx.rs +++ b/big-code-analysis-ast/src/getter/tsx.rs @@ -7,6 +7,7 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { lists: &[Tsx::TypeArguments as u16, Tsx::TypeParameters as u16], openers: &[Tsx::LT as u16], closers: &[Tsx::GT as u16], + is_misparse: never_misparsed, }; impl Getter for TsxCode { @@ -86,7 +87,8 @@ impl Getter for TsxCode { // Java bill theirs; their `*_declaration` wrappers stay unlisted. // // `Type` / `Namespace` / `Module2` / `Declare` / `Global` and the - // `TypeIdentifier` operand: see the TS invocation (#1557). + // `TypeIdentifier` operand: see the TS invocation (#1557), as for + // `export_type_name`. // // `Satisfies` / `Keyof` / `Infer` / `Is` / `Asserts2`, the type-level // operator keywords: see the TS invocation (#1561). @@ -103,6 +105,7 @@ impl Getter for TsxCode { operand_extras: [Identifier2, String2, TypeIdentifier], predefined_void: PredefinedType, generic_angles: GENERIC_ANGLES, + export_type_name: [Type, ExportClause, ExportSpecifier], jsx: [JsxText, HtmlCharacterReference], ); diff --git a/big-code-analysis-ast/src/getter/typescript.rs b/big-code-analysis-ast/src/getter/typescript.rs index 3565d8b6..63e458ab 100644 --- a/big-code-analysis-ast/src/getter/typescript.rs +++ b/big-code-analysis-ast/src/getter/typescript.rs @@ -10,6 +10,7 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { ], openers: &[Typescript::LT as u16], closers: &[Typescript::GT as u16], + is_misparse: never_misparsed, }; impl Getter for TypescriptCode { @@ -94,9 +95,11 @@ impl Getter for TypescriptCode { // `Module` is the `module` declaration node that wraps it, so it // stays unlisted. Each word is contextual, but used as a name // (`let type = 1`, `module.exports`) the grammar emits an - // `identifier`, never the keyword leaf, so no parent gate is - // needed. `type` also bills in `import type` / `export type`, - // where it is the same leaf. + // `identifier`, never the keyword leaf. The one exception is a + // binding named `type` in an export clause (`export { type }`), + // which the grammar recovers as the keyword leaf; `export_type_name` + // leaves that leaf unbilled. `type` also bills in `import type` / + // `export type`, where it is the same leaf. // // `TypeIdentifier` is the operand extra: every type *name* — a // class, interface or alias name, a type parameter, an annotation @@ -137,6 +140,7 @@ impl Getter for TypescriptCode { operand_extras: [TypeIdentifier], predefined_void: PredefinedType, generic_angles: GENERIC_ANGLES, + export_type_name: [Type, ExportClause, ExportSpecifier], ); get_operator!(Typescript); diff --git a/big-code-analysis-ast/src/lang_helpers/bash.rs b/big-code-analysis-ast/src/lang_helpers/bash.rs index f0328d00..c3813554 100644 --- a/big-code-analysis-ast/src/lang_helpers/bash.rs +++ b/big-code-analysis-ast/src/lang_helpers/bash.rs @@ -2,7 +2,7 @@ //! `[ … ]` / `[[ … ]]` test. use crate::Bash; -use crate::node::{Ancestors, Node}; +use crate::node::Node; /// The `&&` / `||` a `[ … ]` connective stands for: `Some(AMPAMP)` for /// `-a` and `Some(PIPEPIPE)` for `-o`, when `op`, a child of `parent`, @@ -31,27 +31,37 @@ pub fn bash_test_connective(op: &Node, parent: &Node, code: &[u8]) -> Option(eq: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - ancestors - .iter(eq) - .map(|(ancestor, _)| ancestor) - .find(|ancestor| { - !matches!( - ancestor.kind(), +pub fn bash_test_eq_count(test: &Node) -> usize { + let mut count = 0; + let mut pending = vec![*test]; + while let Some(expression) = pending.pop() { + for child in expression.children() { + if child.kind_id() == Bash::EQ as u16 { + count += 1; + } else if matches!( + child.kind(), "binary_expression" | "unary_expression" | "parenthesized_expression" - ) - }) - .is_some_and(|ancestor| ancestor.kind_id() == Bash::TestCommand as u16) + ) { + pending.push(child); + } + } + } + count } diff --git a/big-code-analysis-ast/src/lang_helpers/elixir.rs b/big-code-analysis-ast/src/lang_helpers/elixir.rs index 7a200d15..5fff724a 100644 --- a/big-code-analysis-ast/src/lang_helpers/elixir.rs +++ b/big-code-analysis-ast/src/lang_helpers/elixir.rs @@ -93,10 +93,12 @@ pub fn elixir_do_block_call_children<'a>( /// /// An operator token also *names* the operator: `&==/2`, `&and/2` and /// `Kernel.||(a, b)` hold it inside an `operator_identifier`, and the -/// single-token `atom` / `keyword` spell it too. None of those compares, -/// chains or decides anything, and under error recovery the parent can -/// be an `ERROR` with no operands, so every decision metric keys on this -/// parent rather than on the token (#1531, #1534). +/// single-token `atom` / `keyword` spell it too. None of those is a +/// `binary_operator` with operands to compare or chain, and under error +/// recovery the parent can be an `ERROR` with no operands, so every +/// decision metric keys on this parent rather than on the token (#1531, +/// #1534). A remote call that names the operator still *applies* it, +/// though; see [`elixir_remote_call_applies`]. /// /// Matched by rule name rather than by `kind_id`: the grammar aliases /// `binary_operator` to three ids (`Elixir::BinaryOperator` through @@ -111,3 +113,40 @@ pub fn elixir_applying_operator<'a>( .parent(op) .filter(|parent| parent.kind() == "binary_operator") } + +/// Whether `op`, an operator token, is the name of a remote call on +/// `Kernel` with arguments: `Kernel.||(a, b)`, or `x |> Kernel.||(y)`, +/// which the pipe rewrites to it. Such a call expands the `Kernel` macro +/// exactly as `a || b` does, so it short-circuits and decides as the +/// operator does, where a capture (`&Kernel.||/2`, a call with no +/// arguments) only names it. It has no `binary_operator`, so it carries no +/// `left` / `right` operands either. +/// +/// The receiver must spell `Kernel` (or `Elixir.Kernel`): a user module +/// may define `||/2` as an ordinary function, and `Foo.||(a, b)` neither +/// short-circuits nor decides. +/// +/// Matched by rule name: `dot` and `arguments` are aliased to several +/// ids each (grammar-dispatch §1). +#[must_use] +pub fn elixir_remote_call_applies<'a>( + op: &Node<'a>, + code: &[u8], + ancestors: Ancestors<'a, '_>, +) -> bool { + let mut up = ancestors.iter(op).map(|(node, _)| node); + up.next() + .is_some_and(|name| name.kind() == "operator_identifier") + && up.next().is_some_and(|dot| { + dot.kind() == "dot" + && dot.child_by_field_name("left").is_some_and(|receiver| { + matches!( + code.get(receiver.start_byte()..receiver.end_byte()), + Some(b"Kernel" | b"Elixir.Kernel") + ) + }) + }) + && up.next().is_some_and(|call| { + call.kind() == "call" && call.children().any(|child| child.kind() == "arguments") + }) +} diff --git a/big-code-analysis-ast/src/lang_helpers/perl.rs b/big-code-analysis-ast/src/lang_helpers/perl.rs index 1e192f51..dec1db62 100644 --- a/big-code-analysis-ast/src/lang_helpers/perl.rs +++ b/big-code-analysis-ast/src/lang_helpers/perl.rs @@ -55,7 +55,11 @@ pub fn perl_is_and_key_misparse(expr: &Node) -> bool { /// the token; /// - a fat-comma key (`(not => 1)`, `{ not => 1 }`) gives the /// `unary_expression` a real `not 1` gets, with an `ERROR` wrapping -/// the `=>` as the token's next sibling. +/// the `=>` as the token's next sibling; +/// - a fat-comma key after an earlier keyword key in the same list +/// (`f(and => 1, not => 2)`) gives a `not` applied to a zero-width +/// bareword the recovery invents, with the `=>` beside the whole +/// `unary_expression`. A real `not` always has a written operand. /// /// `and`'s test does not carry over: `not` is a prefix operator, so it /// is the first child of its expression whether it is a key or not. The @@ -75,7 +79,10 @@ pub fn perl_not_is_key(not: &Node, parent: &Node) -> bool { .filter(|child| child.is_error() || !child.as_tree_sitter().is_extra()) .skip_while(|child| child.id() != not.id()) .nth(1) - .is_some_and(|next| next.is_error() && next.is_child(Perl::FatComma as u16)) + .is_some_and(|next| { + (next.is_error() && next.is_child(Perl::FatComma as u16)) + || next.start_byte() == next.end_byte() + }) } /// Whether `test`, a Perl `file_handle_operator` whose ancestor chain is @@ -87,7 +94,7 @@ pub fn perl_not_is_key(not: &Node, parent: &Node) -> bool { /// `-foo` before `=>` and alone in a hash subscript — the Tk, CGI and /// Getopt option-hash idiom — so `$h{-foo}` and `(-text => 1)` hold the /// strings `"-foo"` and `"-text"`, and no operator. The grammar emits -/// two shapes: +/// two file-test shapes, which this recognises: /// /// - `-` and a letter glued to more word characters (`-foo`) becomes a /// file test on the bareword `oo`. Perl never reads that as a file @@ -101,8 +108,11 @@ pub fn perl_not_is_key(not: &Node, parent: &Node) -> bool { /// /// The subscript form of the second shape (`$h{-x}`) recovers into an /// `ERROR` that drops the `-x` token altogether, so nothing is left to -/// bill. Remove this, and every call site, once the pin carries the -/// upstream fix. +/// bill. A key whose letter is no file-test letter (`-name`, `-height`) +/// is a third shape, a `-` applied to the bareword, which this does not +/// cover: it still bills the `-` and the word, and ABC counts the word +/// as a call, as every bareword key does. Remove this, and every call +/// site, once the pin carries the upstream fix. #[must_use] pub(crate) fn perl_file_test_is_dash_key<'a>( test: &Node<'a>, @@ -139,7 +149,14 @@ pub fn perl_is_dash_key_word<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) /// whether the keyword is a key, and a key's sign is part of the string /// whether or not a space separates the two. #[must_use] -pub(crate) fn perl_dash_signs_keyword_key(dash: &Node, parent: &Node) -> bool { +pub fn perl_dash_signs_keyword_key(dash: &Node, parent: &Node) -> bool { + // A sign is a prefix `-`, whose parent is the subscript's `ERROR` or + // a `unary_expression`. An infix `-` is a real subtraction even + // before an auto-quoted key (`(1 - and => 2)` subtracts the string), + // and is by far the commonest `-`, so it is answered first. + if parent.kind_id() == Perl::BinaryExpression as u16 { + return false; + } // `$h{-not}` keeps the keyword beside the `-`; `(-not => 1)` nests // it as the first child of the `-`'s operand. next_non_extra(parent, dash).is_some_and(|next| { @@ -148,6 +165,34 @@ pub(crate) fn perl_dash_signs_keyword_key(dash: &Node, parent: &Node) -> bool { }) } +/// Whether `key`, an `and` / `not` key token whose ancestor chain is +/// `ancestors`, is signed by a `-` (`$h{-not}`, `(- and => 1)`), read off +/// the tree as [`perl_dash_signs_keyword_key`] reads it: the `-` sits +/// beside the key, or is the operator of the expression the key leads. +/// The bytes cannot say it: an infix `-` (`$x - and => 2`) or a comment +/// ending in one also precedes a key, and a comment can sit between a +/// sign and its key. +#[must_use] +pub fn perl_key_is_signed<'a>(key: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let mut up = ancestors.iter(key).map(|(ancestor, _)| ancestor); + let Some(parent) = up.next() else { + return false; + }; + let beside = parent + .children() + .filter(|sibling| !sibling.as_tree_sitter().is_extra()) + .take_while(|sibling| sibling.id() != key.id()) + .last(); + if let Some(dash) = beside.filter(|node| node.kind_id() == Perl::DASH as u16) { + return perl_dash_signs_keyword_key(&dash, &parent); + } + up.next().is_some_and(|outer| { + outer.child_by_field_name("operator").is_some_and(|dash| { + dash.kind_id() == Perl::DASH as u16 && perl_dash_signs_keyword_key(&dash, &outer) + }) + }) +} + fn keyword_is_key(token: &Node, parent: &Node) -> bool { match token.kind_id().into() { Perl::And => !perl_and_is_operator(token, parent), @@ -156,12 +201,16 @@ fn keyword_is_key(token: &Node, parent: &Node) -> bool { } } +// The width of a file-test operator token (`-e`, `-x`, …): the grammar +// lexes it as one hidden anonymous `/-[rwxo…]/` token, a `-` and one +// letter, so no node carries its end byte. +const FILE_TEST_TOKEN_LEN: usize = 2; + // Whether `word` is the bareword the grammar split off a `-bareword` -// key: the `-X` file-test token is two bytes, so a word glued to it -// starts right after them. +// key: a word glued to the file-test token starts right after it. fn is_glued_word(word: &Node, test: &Node) -> bool { word.kind_id() == Perl::CallExpressionWithBareword as u16 - && word.start_byte() == test.start_byte() + 2 + && word.start_byte() == test.start_byte() + FILE_TEST_TOKEN_LEN } // Whether the `unary_expression` around `test` is the whole key of a diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 88b58adc..19d7c6c5 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -722,7 +722,10 @@ tokens involved the same way they spell real operators: `enum class`, `data`, `sealed`, the visibility keywords, `open`, `override`, `suspend`, `reified`, `out` and the rest each count once, as the keyword and never as its `modifiers` wrapper. The same words - used as names (`val open = 1`) stay operands. #1557 added C++'s `namespace`, `template`, `typename`, + used as names bill no operator: where the grammar parses the name + (`val open = 1`) it is an operand, and where it cannot (`open = !open` + at the start of a statement, `val suspend = 1`, a parameter named + `vararg`) the word bills nothing. #1557 added C++'s `namespace`, `template`, `typename`, `typedef` and `using` (C and Objective-C bill `typedef` too), so `template ` and `template ` each bill their keyword and score alike. It also added TypeScript's `type`, @@ -749,13 +752,23 @@ tokens involved the same way they spell real operators: `@interface A : NSObject `, and a type passed as a call argument, `f(NSArray)`. A comparison's `<` and `>` and a shift's `<<` and `>>` stay separate operators, and a C++ `>>` that closes two template lists - parses as two `>`, so it bills nothing. Go's generics use square + parses as two `>`, so it bills nothing. The rule follows the parse: + where tree-sitter-cpp reads comparisons as a template argument list, + as in `x = a < b || c > (d)`, their brackets bill as `<>` too, since + only name lookup tells that from a template call spelled the same. + The comparisons Objective-C and C# misread as a call argument's + generic list, `g(ad)` and `F(a < b, c > d)`, keep `<` and `>`. + The exception is C#'s `>=` spelling, `F(a < b, c >= d)`, which the + grammar parses as an assignment and which still bills `<>` and `=`. Go's generics use square brackets and already billed `[]`. Angle brackets around anything other than a generic list still bill `<` and `>`: a Rust `::` path, a C# `delegate*` pointer type and a Kotlin `super`, as well as the JSX, Lua and Perl constructs above. So does a Kotlin generic call with arguments, `id(a)`, - which tree-sitter-kotlin-ng parses as two comparisons (#1394). + which tree-sitter-kotlin-ng parses as two comparisons (#1394), and a + generic inside a Rust macro call, `vec![Vec::::new()]`, whose + token tree tree-sitter-rust leaves unparsed; a `>>` closing two lists + there bills as the shift. Before #1559 each bracket billed under the comparison's own entry, so `Vec` and `a < b` shared one. Every generic now costs one operator occurrence less, so `N1` falls in any code that uses @@ -815,14 +828,16 @@ tokens involved the same way they spell real operators: through a template name (`p->~T()`), which tree-sitter-cpp cannot parse. Another is a statement-position `typename` cast, which made every inserted `;` a second `;` operator. -- **A named operator is one operand.** Elixir names an operator - without applying it in a capture (`&==/2`, `&and/2`) or a remote - call (`Kernel.||(a, b)`), and its grammar wraps the token in an +- **A named operator is one operand.** Elixir names an operator in a + capture (`&==/2`, `&and/2`) and as the function of a remote call + (`Kernel.||(a, b)`), and its grammar wraps the token in an `operator_identifier`. That wrapper is the operand, as `foo` is in `&foo/2`, so the token inside it adds no operator; `&==/2` scores - what `&foo/2` does. A named `&&` / `||` / `and` / `or` is no - decision either (#1534). This differs from a C# `operator <`, whose - `<` has no wrapper and so stays the operator the rule above makes it. + what `&foo/2` does. A captured `&&` / `||` / `and` / `or` is no + decision either (#1534), but the remote call applies the operator, + so `Kernel.||(a, b)` is a cyclomatic decision as `a || b` is. This + differs from a C# `operator <`, whose `<` has no wrapper and so stays + the operator the rule above makes it. ### Derived metrics diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index 28b5114d..cbcf2647 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -12811,6 +12811,51 @@ end ); } + // #1547: `co_return` is a coroutine's `return`, so ABC reads its value + // the same way. Each row's expected value is its `return` twin's. A + // `co_yield` hands a value back without returning, as the JS family's + // `yield` does, and scores no value slot either. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + fn assert_co_return_scores_like_return(lang: LANG) { + let source = |body: &str| format!("task f(bool a) {{ {body} }}\n"); + let rows = [ + ("co_return !a;", "return !a;", 1), + ("co_return /*c*/ !a;", "return /*c*/ !a;", 1), + ("co_return a;", "return a;", 0), + ("co_return;", "return;", 0), + ]; + for (coroutine, twin, expected) in rows { + assert_eq!( + abc_conditions(lang, &source(twin)), + expected, + "{lang:?} `{twin}`" + ); + assert_eq!( + abc_conditions(lang, &source(coroutine)), + expected, + "{lang:?} `{coroutine}`" + ); + } + assert_eq!( + abc_conditions(lang, &source("co_yield !a;")), + 0, + "{lang:?} co_yield" + ); + } + + #[cfg(feature = "cpp")] + #[test] + fn cpp_co_return_value_scores_like_return_1547() { + assert_co_return_scores_like_return(LANG::Cpp); + } + + // Mozcpp owns no file extension, so this is its only coverage. + #[cfg(feature = "mozcpp")] + #[test] + fn mozcpp_co_return_value_scores_like_return_1547() { + assert_co_return_scores_like_return(LANG::Mozcpp); + } + #[cfg(feature = "cpp")] #[test] fn cpp_short_circuit_with_boolean_literal_operand() { diff --git a/src/metrics/abc/bash.rs b/src/metrics/abc/bash.rs index dd08dd26..9ef6d487 100644 --- a/src/metrics/abc/bash.rs +++ b/src/metrics/abc/bash.rs @@ -10,7 +10,7 @@ )] use super::{Abc, Stats}; -use crate::lang_helpers::bash::{bash_eq_is_comparison, bash_test_connective}; +use crate::lang_helpers::bash::{bash_test_connective, bash_test_eq_count}; use crate::*; impl Abc for BashCode { @@ -72,9 +72,10 @@ impl Abc for BashCode { stats.conditions += 1.; } // `=` compares strings inside `[ … ]` / `[[ … ]]`, exactly as - // `==` does, and assigns inside `(( … ))` (#1536). - Bash::EQ if bash_eq_is_comparison(node, ancestors) => { - stats.conditions += 1.; + // `==` does, and assigns inside `(( … ))` (#1536). Each test + // counts its own, once, from the top. + Bash::TestCommand => { + stats.conditions += bash_test_eq_count(node) as f64; } // `<` and `>` are comparisons only inside a `binary_expression`. // The same two tokens spell an I/O redirection (`cmd > out`, diff --git a/src/metrics/abc/cpp.rs b/src/metrics/abc/cpp.rs index e531e266..6753adf3 100644 --- a/src/metrics/abc/cpp.rs +++ b/src/metrics/abc/cpp.rs @@ -369,7 +369,11 @@ impl Abc for CppCode { IfStatement | WhileStatement | DoStatement => { cpp_count_condition_slot(node, &mut stats.conditions); } - ReturnStatement => { + // `co_return` is a coroutine's `return` (#1547), so its value + // is read the same way: `co_return !x` counts one. `co_yield` + // hands back a value without returning, as the JS family's + // `yield` does, and like it scores no value slot. + ReturnStatement | CoReturnStatement => { cpp_count_return(node, &mut stats.conditions); } // `f(!a, !b)` — argument list walker. Two aliases — diff --git a/src/metrics/abc/mozcpp.rs b/src/metrics/abc/mozcpp.rs index d5f32b2f..6aba4411 100644 --- a/src/metrics/abc/mozcpp.rs +++ b/src/metrics/abc/mozcpp.rs @@ -124,7 +124,9 @@ impl Abc for MozcppCode { IfStatement | WhileStatement | DoStatement => { cpp_count_condition_slot(node, &mut stats.conditions); } - ReturnStatement => { + // `co_return` reads its value as `return` does; the + // derivation is on the Cpp twin (#1547). + ReturnStatement | CoReturnStatement => { cpp_count_return(node, &mut stats.conditions); } // `f(!a, !b)` — argument list walker. Two aliases — diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index 3f514094..f87c8804 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -11,9 +11,11 @@ use super::{ Abc, Stats, count_boolean_slot, count_each_operand, count_negated_operand, is_operand, - last_operand, wrapped_operand, + last_operand, peel, wrapped_operand, +}; +use crate::lang_helpers::perl::{ + perl_and_is_operator, perl_dash_signs_keyword_key, perl_is_dash_key_word, perl_not_is_key, }; -use crate::lang_helpers::perl::{perl_and_is_operator, perl_is_dash_key_word, perl_not_is_key}; use crate::*; // Fitzpatrick's ABC rules adapted for Perl. @@ -75,6 +77,12 @@ use crate::*; // parses as that same `unary_expression`, with the `=>` in an `ERROR` // beside the token. It is the list `("not", !$a)`, not a negation, so // it peels like one: to its last element, the value, proving nothing. +// FIXME(#1539 upstream): an auto-quoted `and` key (`f(and => !$a)`) is +// the same shape led by `and`, and peels the same way; a real `and` +// has a left operand, so `perl_and_is_operator` tells the two apart. +// FIXME(#1545 upstream): a `-and` / `-not` key nests that shape as the +// operand of a `-` (`f(-not => !$a)`); the sign is part of the key, so +// the peel steps through it to the key. // // No operand is read by index: a comment may sit before or after it // (`(# c` / `! # c`, #1455). @@ -91,6 +99,12 @@ fn perl_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { node.child_by_field_name("operator") .and_then(|op| match op.kind_id().into() { P::Not if perl_not_is_key(&op, node) => last_operand(node).map(|o| (o, false)), + P::And if !perl_and_is_operator(&op, node) => { + last_operand(node).map(|o| (o, false)) + } + P::DASH if perl_dash_signs_keyword_key(&op, node) => { + wrapped_operand(node).map(|o| (o, false)) + } P::BANG | P::Not => wrapped_operand(node).map(|o| (o, true)), _ => None, }) @@ -153,13 +167,18 @@ fn perl_condition_scores_itself(expr: &Node) -> bool { // `$x !~ s///r` and `$x !~ tr///r` parse the same way but are Perl // compile errors ("Using !~ with s///r doesn't make sense"); valid input // cannot tell how they score, so no test pins them (grammar-dispatch §6). +// +// Only a `binary_expression` binds: under error recovery the token's +// parent can be an `ERROR` holding the whole file, whose last operand is +// no right operand, and reading it per token was quadratic. fn perl_binding_rewrites(binding: &Node) -> bool { - last_operand(binding).is_some_and(|rhs| { - matches!( - rhs.kind_id().into(), - Perl::SubstitutionPatternS | Perl::TransliterationTrOrY - ) - }) + binding.kind_id() == Perl::BinaryExpression as u16 + && last_operand(binding).is_some_and(|rhs| { + matches!( + rhs.kind_id().into(), + Perl::SubstitutionPatternS | Perl::TransliterationTrOrY + ) + }) } // Scores one boolean slot — an `if` / `elsif` / `unless` / `while` / @@ -217,50 +236,71 @@ fn perl_count_slot(slot: Option, conditions: &mut f64) { // suffices: those operators occur only in `binary_expression`, apart // from the readline `` brackets, whose interior is never an // expression. +// +// Two guards keep the walk linear, and neither moves a count. A slot +// that would pay nothing anyway (its occupant scores itself — on a +// left-nested `$a || $b || …` spine, every slot but the bottom one) is +// answered at once, so each spine is climbed once, from its bottom; and +// only a `binary_expression`, or an `ERROR` recovered from one, can hold +// a chaining operator, so a block, list or file the climb stops at is +// not scanned child by child. Without them every spine node climbed to +// the top and every chain scanned its statement list, both quadratic. fn perl_misparse_absorbs_leading_operand<'a>( node: &Node<'a>, - mut ancestors: impl Iterator>, + ancestors: impl Iterator>, ) -> bool { use Perl as P; + if perl_leading_slot(node) + .is_none_or(|lead| perl_condition_scores_itself(&peel(&lead, perl_wrapper_operand).0)) + { + return false; + } let mut child = *node; - ancestors - .find(|parent| { - let leading = match parent.kind_id().into() { - P::TernaryExpression => parent.child_by_field_name("condition"), - _ if perl_is_logical_chain(parent) => wrapped_operand(parent), - _ => None, - }; - let on_spine = leading.is_some_and(|lead| lead.id() == child.id()); - if on_spine { - child = *parent; - } - !on_spine - }) - .is_some_and(|parent| { - parent - .children() - .take_while(|sibling| sibling.id() != child.id()) - .filter(|sibling| !sibling.as_tree_sitter().is_extra()) - .last() - .is_some_and(|operator| { - matches!( - operator.kind_id().into(), - P::LT - | P::GT - | P::LTEQ - | P::GTEQ - | P::Lt - | P::Gt - | P::Le - | P::Ge - | P::EQEQ - | P::BANGEQ - | P::Eq - | P::Ne - ) - }) - }) + for parent in ancestors { + if perl_leading_slot(&parent).is_some_and(|lead| lead.id() == child.id()) { + child = parent; + continue; + } + if parent.kind_id() != P::BinaryExpression as u16 && !parent.is_error() { + return false; + } + return parent + .children() + .take_while(|sibling| sibling.id() != child.id()) + .filter(|sibling| !sibling.as_tree_sitter().is_extra()) + .last() + .is_some_and(|operator| { + matches!( + operator.kind_id().into(), + P::LT + | P::GT + | P::LTEQ + | P::GTEQ + | P::Lt + | P::Gt + | P::Le + | P::Ge + | P::EQEQ + | P::BANGEQ + | P::Eq + | P::Ne + ) + }); + } + false +} + +// The leading boolean slot of a spine node — a ternary's condition or a +// logical chain's first operand — or `None` for any other node. +fn perl_leading_slot<'a>(node: &Node<'a>) -> Option> { + if node.kind_id() == Perl::TernaryExpression as u16 { + node.child_by_field_name("condition") + } else if perl_is_logical_chain(node) { + wrapped_operand(node) + } else { + None + } } // A `&&` / `||` / `//` / `and` / `or` chain. tree-sitter-perl spells diff --git a/src/metrics/bash_test_connective_tests.rs b/src/metrics/bash_test_connective_tests.rs index 2bbbf732..fe1e3710 100644 --- a/src/metrics/bash_test_connective_tests.rs +++ b/src/metrics/bash_test_connective_tests.rs @@ -78,7 +78,7 @@ fn bash_test_connectives_score_like_their_twins_1536() { // `-a` and `-o` key to the symbols they stand for: a run of one // is a single sequence. The grammar nests `=`, `==` and `!=` // around the connectives (`"$a" = (1 -o "$b") = …`), so these - // rows also pin `bash_sequence_end`; `[[ … = … || … ]]` scored 3 + // rows also pin `ComparisonRuns`; `[[ … = … || … ]]` scored 3 // before #1536 for the same reason. ( r#"if [ "$a" = 1 -o "$b" = 2 -o "$c" = 3 ]; then :; fi"#, @@ -160,3 +160,63 @@ fn bash_test_connective_gates_keep_their_controls_1536() { ); } } + +/// An outer comparison run resumes after a nested one: past the `&&` +/// group, `"$c" = 3` continues the `||` run `"$b" = 2` restarted, so the +/// two are one sequence. The mis-nested tree scores a mixed run in +/// textual order (`ComparisonRuns`), one more than the `==` twin's 3, so +/// the value here is the one the ancestor climb `ComparisonRuns` +/// replaced scored; a tracker that forgot the outer run at the group +/// scores 5. +#[cfg(feature = "bash")] +#[test] +fn bash_comparison_run_resumes_after_a_nested_run_1536() { + assert_eq!( + measure( + r#"if [[ "$a" = 1 || ( "$x" = 5 && "$y" = 6 ) || "$b" = 2 || "$c" = 3 || "$d" = 4 ]]; then :; fi"# + ), + [8, 7, 4] + ); +} + +/// `=~` shares `=`'s precedence, so tree-sitter-bash mis-nests a run of +/// regex matches the same way (`"$a" =~ ($re || "$b") =~ …`), and the run +/// must stay one sequence, as its `==` twin's is. The other two rows pin +/// what the run tracking and the `=` count reproduce: a run that resumes +/// after a nested one keeps its sequences, and an assignment inside a +/// test's `$(( … ))` is no comparison. +#[cfg(feature = "bash")] +#[test] +fn bash_comparison_runs_score_like_their_twins_1536() { + // `(body, twin, decisions)`; `twin` spells the same test another + // way and must agree. + let rows: [(&str, &str, Decisions); 3] = [ + ( + r#"if [[ "$a" =~ $re || "$b" =~ $re || "$c" =~ $re ]]; then :; fi"#, + r#"if [[ "$a" == $re || "$b" == $re || "$c" == $re ]]; then :; fi"#, + [5, 4, 2], + ), + ( + "if [[ ( $a = 1 || $b = 2 ) && $c = 3 || $d = 4 ]]; then :; fi", + "if [[ ( $a == 1 || $b == 2 ) && $c == 3 || $d == 4 ]]; then :; fi", + [6, 5, 4], + ), + ( + "if [[ $(( x = 1 )) -eq 1 ]]; then :; fi", + "if [[ $(( x + 1 )) -eq 1 ]]; then :; fi", + [3, 2, 1], + ), + ]; + for (body, twin, decisions) in rows { + assert_eq!( + measure(twin), + decisions, + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure(body), + decisions, + "`{body}` must score like `{twin}`" + ); + } +} diff --git a/src/metrics/cognitive.rs b/src/metrics/cognitive.rs index 05283c9a..c79ad483 100644 --- a/src/metrics/cognitive.rs +++ b/src/metrics/cognitive.rs @@ -55,6 +55,7 @@ pub struct Stats { total_space_functions: usize, boolean_seq: BoolSequence, constraint: ConstraintReach, + bash_runs: bash::ComparisonRuns, } impl Default for Stats { @@ -68,6 +69,7 @@ impl Default for Stats { total_space_functions: 1, boolean_seq: BoolSequence::default(), constraint: ConstraintReach::default(), + bash_runs: bash::ComparisonRuns::default(), } } } diff --git a/src/metrics/cognitive/bash.rs b/src/metrics/cognitive/bash.rs index 5ab6f576..751501ed 100644 --- a/src/metrics/cognitive/bash.rs +++ b/src/metrics/cognitive/bash.rs @@ -18,26 +18,80 @@ use crate::lang_helpers::bash::bash_test_connective; // comparison swallow the connectives around it, so // `[ "$a" = 1 -o "$b" = 2 -o "$c" = 3 ]` parses as // `"$a" = (1 -o "$b") = (2 -o "$c") = 3` (and `[[ … = … || … ]]` the -// same). The two `-o` then sit in sibling subtrees, neither inside the -// other's span, and the run of one operation scored two sequences. A -// connective's expression is an operand of a comparison only in that -// mis-nested tree, so the sequence it belongs to ends where the -// outermost such comparison does. A correct tree (`==` inside `[[ … ]]`, -// `-eq`) never puts a connective under a comparison, so this returns -// `node`'s own end there. The mis-nested tree has lost precedence, so a -// mixed run is scored in textual order: `[[ "$a" = 1 || "$b" = 2 && -// "$c" = 3 || "$d" = 4 ]]` scores three sequences where the `==` twin, -// nested `(a || (b && c)) || d`, scores two. -fn bash_sequence_end(node: &Node, ancestors: Ancestors) -> usize { - ancestors - .iter(node) - .map(|(ancestor, _)| ancestor) - .take_while(|ancestor| { - ancestor.kind_id() == Bash::BinaryExpression3 as u16 - && ancestor.wraps_any(&[Bash::EQ as u16, Bash::EQEQ as u16, Bash::BANGEQ as u16]) - }) - .last() - .map_or(node.end_byte(), |outermost| outermost.end_byte()) +// same, as `=~`, which shares `=`'s precedence). The two `-o` then sit +// in sibling subtrees, neither inside the other's span, and the run of +// one operation scored two sequences. A connective's expression is an +// operand of such a comparison only in that mis-nested tree, so the +// sequence it belongs to ends where the outermost comparison of the run +// does; where no comparison holds the connective, its own node's end. +// The mis-nested tree has lost precedence, so a mixed run is scored in +// textual order: `[[ "$a" = 1 || "$b" = 2 && "$c" = 3 || "$d" = 4 ]]` +// scores three sequences where the `==` twin, nested +// `(a || (b && c)) || d`, scores two. `[ … ]` gives every `-x` test one +// precedence, so a mixed `-a` / `-o` run over `-eq` tests is scored in +// textual order too, though a run of one connective there nests down a +// single spine and needs nothing from this. +const RUN_COMPARISONS: [u16; 4] = [ + Bash::EQ as u16, + Bash::EQEQ as u16, + Bash::BANGEQ as u16, + Bash::EQTILDE as u16, +]; + +/// The comparison runs the walk is inside, innermost last. A run member +/// is a `BinaryExpression3` whose operator is one of +/// [`RUN_COMPARISONS`]; each records its span and the end of the +/// outermost member of its run, inherited when its parent is a member +/// too. +/// +/// The walk is pre-order, so a member is visited before everything under +/// it, and one whose span does not hold the node being visited is +/// finished. Climbing from every `BinaryExpression3` to the top of its run +/// instead made a long `[ … -o … ]` or `[[ … = … && … ]]` test quadratic. +/// A stack rather than one slot, because a run can sit in a parenthesised +/// operand of another and the outer run resumes after it: +/// `[[ $a = 1 || ( $x = 5 && $y = 6 ) || $b = 2 || $c = 3 ]]` keeps +/// `$c = 3` in the `||` sequence `$b = 2` opened. Walk state: never +/// serialized or merged. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(super) struct ComparisonRuns(Vec); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct RunMember { + id: usize, + start: usize, + end: usize, + run_end: usize, +} + +impl ComparisonRuns { + /// The byte a boolean sequence under `node`, a `BinaryExpression3` + /// whose parent is `parent`, extends to: the end of the run `parent` + /// is a member of, else `node`'s own end. Records `node` when it is a + /// member itself. + fn sequence_end(&mut self, node: &Node, parent: Option) -> usize { + let (start, end) = (node.start_byte(), node.end_byte()); + while self + .0 + .last() + .is_some_and(|member| start < member.start || member.end < end) + { + self.0.pop(); + } + let run_end = match (parent, self.0.last()) { + (Some(parent), Some(member)) if member.id == parent.id() => member.run_end, + _ => end, + }; + if node.wraps_any(&RUN_COMPARISONS) { + self.0.push(RunMember { + id: node.id(), + start, + end, + run_end, + }); + } + run_end + } } impl Cognitive for BashCode { @@ -87,7 +141,7 @@ impl Cognitive for BashCode { compute_booleans(node, stats, AMPAMP, PIPEPIPE); } BinaryExpression3 => { - let end = bash_sequence_end(node, ancestors); + let end = stats.bash_runs.sequence_end(node, ancestors.parent(node)); compute_booleans_by_node(node, end, stats, |child| match child.kind_id().into() { AMPAMP | PIPEPIPE => Some(child.kind_id()), _ => bash_test_connective(child, node, code).map(|op| op as u16), diff --git a/src/metrics/cyclomatic/elixir.rs b/src/metrics/cyclomatic/elixir.rs index 2ee64e10..30793877 100644 --- a/src/metrics/cyclomatic/elixir.rs +++ b/src/metrics/cyclomatic/elixir.rs @@ -116,12 +116,19 @@ impl Cyclomatic for ElixirCode { stats.cyclomatic_modified += 1.; } // Short-circuit booleans add a decision point in both - // metrics — where applied. `&and/2`, `&||/2` and - // `Kernel.||(a, b)` only name the operator and decide - // nothing (#1534); ABC and cognitive gate on the same parent. + // metrics — where applied. `&and/2` and `&||/2` only name the + // operator and decide nothing (#1534); ABC and cognitive gate + // on the same parent. A remote call naming it, + // `Kernel.||(a, b)`, applies the macro and decides like + // `a || b`; a user module's `Foo.||(a, b)` is an ordinary + // call and does not. ABC and cognitive still score the + // `Kernel` call 0, as they did before #1534. E::AMPAMP | E::PIPEPIPE | E::And | E::Or if crate::lang_helpers::elixir::elixir_applying_operator(node, ancestors) - .is_some() => + .is_some() + || crate::lang_helpers::elixir::elixir_remote_call_applies( + node, code, ancestors, + ) => { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; diff --git a/src/metrics/elixir_named_operator_tests.rs b/src/metrics/elixir_named_operator_tests.rs index da0abebf..1a8153f1 100644 --- a/src/metrics/elixir_named_operator_tests.rs +++ b/src/metrics/elixir_named_operator_tests.rs @@ -1,12 +1,14 @@ //! A *named* Elixir operator scores like any other name (#1534). //! -//! `&==/2`, `&and/2` and `Kernel.||(a, b)` name an operator without -//! applying it: the grammar wraps the token in an `operator_identifier`. -//! Halstead billed both the wrapper (an operand) and the token inside it -//! (an operator), and cyclomatic counted a named `&&` / `||` / `and` / -//! `or` as a decision. Every row pairs the fixture with a twin naming an -//! ordinary function, which is the oracle; the twin's numbers are -//! asserted outright too, so a row cannot pass by both drifting. +//! `&==/2`, `&and/2` and `Kernel.||(a, b)` name an operator: the grammar +//! wraps the token in an `operator_identifier`. Halstead billed both the +//! wrapper (an operand) and the token inside it (an operator), and +//! cyclomatic counted a captured `&&` / `||` / `and` / `or` as a decision. +//! Every row pairs the fixture with a twin naming an ordinary function, +//! which is the oracle; the twin's numbers are asserted outright too, so +//! a row cannot pass by both drifting. A remote call (`Kernel.||(a, b)`) +//! still applies the operator, so it keeps the decision; it is in the +//! applied table below. use std::cell::Cell; @@ -58,10 +60,7 @@ fn names_an_operator(source: &str) -> bool { std::path::Path::new("foo.exs"), None, ); - parser - .root() - .preorder() - .any(|node| node.kind_id() == Elixir::OperatorIdentifier as u16) + crate::test_support::ast_has_kind_id(&parser, Elixir::OperatorIdentifier as u16) } /// Each named operator bills one operand — its `operator_identifier` — @@ -83,7 +82,8 @@ fn elixir_named_operator_scores_like_a_named_function() { ("a = &foo/1", "a = &!/1", [3, 3, 3, 3]), ("a = &foo/2", "a = &+/2", [3, 3, 3, 3]), ("a = Kernel.foo(x, y)", "a = Kernel.==(x, y)", [4, 4, 5, 5]), - ("a = Kernel.foo(x, y)", "a = Kernel.||(x, y)", [4, 4, 5, 5]), + // A capture through the module names the operator too. + ("a = &Kernel.foo/2", "a = &Kernel.||/2", [4, 4, 4, 4]), // `..` is a childless `operator_identifier`: the wrapper is the // only node that can bill it, which is why it is the keeper. ("a = foo", "a = ..", [1, 1, 2, 2]), @@ -115,17 +115,28 @@ fn elixir_named_operators_are_distinct_operands() { } /// An applied operator keeps its operator and its operands, and a -/// short-circuit one keeps its decision in both tiers. +/// short-circuit one keeps its decision in both tiers. A remote call +/// naming a short-circuit operator, directly or through a pipe, expands +/// the `Kernel` macro and so decides as `x || y` does, while billing its +/// name as the operand a remote call's function name is. The same call on +/// a user module (`Foo.||`) is an ordinary function call and decides +/// nothing. #[cfg(feature = "elixir")] #[test] fn elixir_applied_operator_keeps_its_operator() { // `(source, halstead, cyclomatic)`. - let rows: [(&str, Halstead, Cyclomatic); 5] = [ + let rows: [(&str, Halstead, Cyclomatic); 11] = [ ("a = x == y", [2, 2, 3, 3], [1, 1]), ("a = x in y", [2, 2, 3, 3], [1, 1]), ("a = not x", [2, 2, 2, 2], [1, 1]), ("a = x and y", [2, 2, 3, 3], [2, 2]), ("a = x || y", [2, 2, 3, 3], [2, 2]), + ("a = Kernel.||(x, y)", [4, 4, 5, 5], [2, 2]), + ("a = Kernel.&&(x, y)", [4, 4, 5, 5], [2, 2]), + ("a = x |> Kernel.||(y)", [4, 4, 5, 5], [2, 2]), + ("a = Elixir.Kernel.||(x, y)", [4, 4, 5, 5], [2, 2]), + ("a = Foo.||(x, y)", [4, 4, 5, 5], [1, 1]), + ("a = x |> Foo.||(y)", [4, 4, 5, 5], [1, 1]), ]; for (source, halstead, cyclomatic) in rows { assert_eq!(measure(source), (halstead, cyclomatic), "`{source}`"); diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index a9b4519b..57d87134 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -4758,6 +4758,73 @@ mod tests { ); } + /// Asserts a grammar's wrapped generic-list opener, `LT2`, never + /// reaches a node, so the `LT` its getter pairs as `<>` is the only + /// `<` a list opens with (#1559, grammar-dispatch §2). `lists` are the + /// lists the wrapped token opens; the positive control counts their + /// `LT` openers, so the assertion cannot pass on a fixture that + /// stopped producing those lists. + #[cfg(any(feature = "groovy", feature = "objc", feature = "rust"))] + #[track_caller] + fn assert_generic_opener_folds_to_lt( + source: &str, + [lt, lt2]: [u16; 2], + lists: &[&str], + openers: usize, + label: &str, + ) { + let parser = P::new(source.as_bytes().to_vec(), &PathBuf::from("foo"), None); + assert!( + !ast_has_kind_id(&parser, lt2), + "{label}::LT2 must stay collapsed to {label}::LT", + ); + let found = parser + .root() + .preorder() + .filter(|n| n.kind_id() == lt && n.parent().is_some_and(|p| lists.contains(&p.kind()))) + .count(); + assert_eq!( + found, openers, + "{label}: each generic-list opener must be an `LT`" + ); + } + + #[cfg(feature = "rust")] + #[test] + fn rust_generic_opener_alias_never_reaches_kind_id() { + assert_generic_opener_folds_to_lt::( + "fn u(x: Vec) -> impl Sized + use { x }\n", + [Rust::LT as u16, Rust::LT2 as u16], + &["type_arguments", "use_bounds"], + 2, + "Rust", + ); + } + + #[cfg(feature = "groovy")] + #[test] + fn groovy_generic_opener_alias_never_reaches_kind_id() { + assert_generic_opener_folds_to_lt::( + "class A { List v }\n", + [Groovy::LT as u16, Groovy::LT2 as u16], + &["type_parameters", "type_arguments"], + 2, + "Groovy", + ); + } + + #[cfg(feature = "objc")] + #[test] + fn objc_generic_opener_alias_never_reaches_kind_id() { + assert_generic_opener_folds_to_lt::( + "int f(void) { return g(NSArray); }\n", + [Objc::LT as u16, Objc::LT2 as u16], + &["argument_list"], + 1, + "Objc", + ); + } + #[cfg(feature = "perl")] #[test] fn perl_readline_brackets_are_operators() { @@ -6372,8 +6439,10 @@ end", /// #1558: Kotlin bills its modifier keywords as C# bills its own, /// one family per row, each keyword the leaf and never - /// its `*_modifier` wrapper. The last row is the control: the same - /// words used as names are operands, and add no operator. + /// its `*_modifier` wrapper. The "names" row is the control: the same + /// words used as names are operands, and add no operator. The rows + /// after it are names the grammar fails to parse, whose recovery keeps + /// the keyword leaf, and their control. #[cfg(feature = "kotlin")] #[test] fn kotlin_modifiers_are_operators_1558() { @@ -6441,6 +6510,42 @@ end", "val open = 1\nval out = 2\nval value = 3\n", [2, 6], ), + // The rows below are names the grammar fails to parse. Its + // recovery keeps the keyword leaf, which must bill nothing; + // each expected value is what the source scored before #1558. + // `class`, `{}` x2, `var`, `=` x2, `fun`, `()`: `open = !open` + // recovers into an `ERROR` holding `inheritance_modifier`. + ( + "name stranded in an ERROR", + "class Drawer {\n var open = false\n fun toggle() { open = !open }\n}\n", + [6, 8], + ), + // `val`: `suspend` never parses as a name. + ("suspend as a name", "val suspend = 1\n", [1, 1]), + // `fun`, `()`, `:` x2, `=`: a modifier list followed by an + // inserted zero-width parameter name. + ( + "parameter named vararg", + "fun f(vararg: Int): Int = vararg\n", + [4, 5], + ), + // `class`, `()`, `:` + ("class parameter named open", "class C(open: Int)\n", [3, 3]), + // `val`, `=`, `{}`, `:` + ( + "lambda parameter named open", + "val l = { open: Int -> open }\n", + [4, 4], + ), + // Control for the gate: a real `suspend` in a function type's + // `type_modifiers` and in a declaration's `modifiers` bills + // twice. `val`, `:`, `suspend` x2, `()` x2, `->`, `=`, `{}` x2, + // `fun`. + ( + "suspend modifiers", + "val f: suspend () -> Unit = {}\nsuspend fun g() {}\n", + [8, 11], + ), ]; assert_operator_rows::("foo.kt", ROWS); } @@ -6524,8 +6629,12 @@ end", } /// #1557: the TypeScript declaration keywords #1552 left out. The - /// last row pins that each word, used as a *name*, stays an operand: - /// the grammar emits an `identifier` there, never the keyword leaf. + /// "keywords as names" row pins that each word, used as a *name*, + /// stays an operand: the grammar emits an `identifier` there. The two + /// exported-binding rows are the one place it emits the `type` leaf + /// for a name, which must bill nothing; the last two rows are their + /// controls, a type-only specifier and re-export whose `type` still + /// bills. #[cfg(feature = "typescript")] const TS_DECLARATION_KEYWORD_ROWS: &[(&str, &str, [u64; 2])] = &[ // `type`, `=`, `number`, `;` @@ -6554,6 +6663,34 @@ end", "let type = 1;\nmodule.exports = declare;\n", [4, 6], ), + // `const`, `=`, `;` twice, `export`, `{}` — no keyword. The + // grammar recovers an exported binding named `type` as the + // keyword leaf, under an `ERROR` or beside one in the specifier. + ( + "exported binding named type", + "const type = 1;\nexport { type };\n", + [5, 6], + ), + ( + "renamed exported binding named type", + "const type = 1;\nexport { type as kind };\n", + [5, 6], + ), + // `export`, `{}`, `type`, `from`, `;` — a type-only specifier + // parses cleanly, so its keyword still bills. + ( + "type-only export specifier", + "export { type Foo } from \"m\";\n", + [5, 5], + ), + // `export`, `type`, `*`, `from`, `;` — the pinned grammar fails + // TS 5.0's type-only re-export into an `ERROR` that hangs from the + // statement, not from an export clause, and its keyword bills. + ( + "type-only re-export", + "export type * from \"./types\";\n", + [5, 5], + ), ]; /// #1557: a TypeScript type name is an operand, as C#, Rust and the @@ -10931,15 +11068,18 @@ S s; int a = s[1]; auto t = co_await s; assert_halstead_counts::(CPP_OPERATOR_NAMES, "ops.cpp", [17, 40, 6, 9], label); // `operator[]` / `operator()` are kinds of their own, so they sit - // beside the subscript `[` and the parameter-list `(` as a second - // `[]` / `()` entry — two operators, as Ruby's `def [](i)` is. + // beside the subscript `[` and the parameter-list `(` as two more + // operators, each rendered as the operator it names so that every + // listed entry is distinct. let ops = ops_of::(CPP_OPERATOR_NAMES, "ops.cpp"); for (name, entries) in [ ("operator", 1), ("co_await", 1), ("\"\"", 1), - ("[]", 2), - ("()", 2), + ("operator[]", 1), + ("operator()", 1), + ("[]", 1), + ("()", 1), ] { assert_eq!( ops.operators.iter().filter(|o| *o == name).count(), @@ -10995,9 +11135,11 @@ S s; int a = s[1]; auto t = co_await s; /// an operand-classified zero-width node; before the fix every row /// billed `""`. C#'s and Elixir's are not flagged MISSING, which is /// why the guard keys on width: an `is_missing()` guard leaves those - /// two rows billing `""`. The rows pin no counts, since the recovery trees - /// are the grammars' to change (grammar-dispatch §6); - /// `cpp_missing_tokens_in_valid_code_are_not_billed` anchors two. + /// two rows billing `""`. Each row names its zero-width operand's + /// kind and flag, so the two unflagged rows keep discriminating. The + /// rows pin no counts, since the recovery trees are the grammars' to + /// change (grammar-dispatch §6); the operator half, an inserted `;` or + /// `)`, is anchored by `cpp_missing_tokens_in_valid_code_are_not_billed`. #[cfg(any( feature = "bash", feature = "c", @@ -11024,77 +11166,161 @@ S s; int a = s[1]; auto t = co_await s; ))] #[test] fn zero_width_nodes_bill_neither_halstead_half() { - fn has_zero_width_node(node: tree_sitter::Node) -> bool { - node.start_byte() == node.end_byte() || { - let mut cursor = node.walk(); - node.children(&mut cursor).any(has_zero_width_node) - } - } - - let cases: &[(LANG, &str, &str)] = &[ + // `(language, file, source, kind, is_missing)`: the last two name + // the zero-width operand each fixture's recovery inserts, so a + // grammar bump that recovers some other way fails the row rather + // than passing it on an unrelated zero-width node. + let cases: &[(LANG, &str, &str, &str, bool)] = &[ #[cfg(feature = "bash")] - (LANG::Bash, "foo.sh", "echo a $( )\n"), + (LANG::Bash, "foo.sh", "echo a $( )\n", "word", true), #[cfg(feature = "c")] - (LANG::C, "foo.c", "int x; void () { x = 1; }\n"), + ( + LANG::C, + "foo.c", + "int x; void () { x = 1; }\n", + "identifier", + true, + ), #[cfg(feature = "cpp")] - (LANG::Cpp, "foo.cpp", "void f(S *s) {\n s->~D();\n}\n"), + ( + LANG::Cpp, + "foo.cpp", + "void f(S *s) {\n s->~D();\n}\n", + "identifier", + true, + ), #[cfg(feature = "csharp")] - (LANG::Csharp, "foo.cs", "class A { int = 1; }\n"), + ( + LANG::Csharp, + "foo.cs", + "class A { int = 1; }\n", + "identifier", + false, + ), #[cfg(feature = "elixir")] - (LANG::Elixir, "foo.ex", "x = a.\n"), + (LANG::Elixir, "foo.ex", "x = a.\n", "identifier", false), #[cfg(feature = "go")] - (LANG::Go, "foo.go", "package m\nfunc f() { a.() }\n"), + ( + LANG::Go, + "foo.go", + "package m\nfunc f() { a.() }\n", + "type_identifier", + true, + ), #[cfg(feature = "groovy")] - (LANG::Groovy, "foo.groovy", "def x = a.\n"), + ( + LANG::Groovy, + "foo.groovy", + "def x = a.\n", + "identifier", + true, + ), #[cfg(feature = "irules")] ( LANG::Irules, "foo.irule", "when HTTP_REQUEST { set x [expr {1 + }] }\n", + "number", + true, ), #[cfg(feature = "java")] - (LANG::Java, "Foo.java", "class A { int = 1; }\n"), + ( + LANG::Java, + "Foo.java", + "class A { int = 1; }\n", + "identifier", + true, + ), #[cfg(feature = "javascript")] - (LANG::Javascript, "foo.js", "let {a: } = b;\n"), + ( + LANG::Javascript, + "foo.js", + "let {a: } = b;\n", + "identifier", + true, + ), #[cfg(feature = "kotlin")] - (LANG::Kotlin, "foo.kt", "fun f(: Int) {}\n"), + ( + LANG::Kotlin, + "foo.kt", + "fun f(: Int) {}\n", + "identifier", + true, + ), #[cfg(feature = "lua")] - (LANG::Lua, "foo.lua", "x = a.\n"), + (LANG::Lua, "foo.lua", "x = a.\n", "identifier", true), #[cfg(feature = "mozcpp")] ( LANG::Mozcpp, "foo.cpp", "void f(S *s) {\n s->~D();\n}\n", + "identifier", + true, ), #[cfg(feature = "mozjs")] - (LANG::Mozjs, "foo.jsm", "let {a: } = b;\n"), + ( + LANG::Mozjs, + "foo.jsm", + "let {a: } = b;\n", + "identifier", + true, + ), #[cfg(feature = "objc")] - (LANG::Objc, "foo.m", "@interface A\n- (void)f:;\n@end\n"), + ( + LANG::Objc, + "foo.m", + "@interface A\n- (void)f:;\n@end\n", + "identifier", + true, + ), #[cfg(feature = "perl")] - (LANG::Perl, "foo.pl", "sub { $x = ; }\n"), + (LANG::Perl, "foo.pl", "sub { $x = ; }\n", "identifier", true), #[cfg(feature = "php")] - (LANG::Php, "foo.php", ";\n"), + (LANG::Php, "foo.php", ";\n", "name", true), #[cfg(feature = "python")] - (LANG::Python, "foo.py", "for in x:\n pass\n"), + ( + LANG::Python, + "foo.py", + "for in x:\n pass\n", + "identifier", + true, + ), #[cfg(feature = "ruby")] - (LANG::Ruby, "foo.rb", "x = a.\n"), + (LANG::Ruby, "foo.rb", "x = a.\n", "identifier", true), #[cfg(feature = "rust")] - (LANG::Rust, "foo.rs", "fn f() { let x = a.; }\n"), + ( + LANG::Rust, + "foo.rs", + "fn f() { let x = a.; }\n", + "field_identifier", + true, + ), #[cfg(feature = "tcl")] - (LANG::Tcl, "foo.tcl", "puts [expr {1 + }]\n"), + (LANG::Tcl, "foo.tcl", "puts [expr {1 + }]\n", "number", true), #[cfg(feature = "typescript")] - (LANG::Typescript, "foo.ts", "let {a: } = b;\n"), + ( + LANG::Typescript, + "foo.ts", + "let {a: } = b;\n", + "identifier", + true, + ), #[cfg(feature = "typescript")] - (LANG::Tsx, "foo.tsx", "let {a: } = b;\n"), + (LANG::Tsx, "foo.tsx", "let {a: } = b;\n", "identifier", true), ]; crate::test_support::assert_fixtures_present(cases); let mut billed = Vec::new(); - for (lang, file, source) in cases { + for (lang, file, source, kind, missing) in cases { let ast = crate::test_support::parse_named(*lang, file, source); assert!( - has_zero_width_node(ast.as_tree_sitter().root_node()), - "{lang:?}: `{source}` no longer recovers with a zero-width node; find another fixture" + ast.root_node().preorder().any(|node| { + node.start_byte() == node.end_byte() + && node.kind() == *kind + && node.is_missing() == *missing + }), + "{lang:?}: `{source}` no longer recovers with a zero-width `{kind}` \ + (is_missing: {missing}); find another fixture" ); let ops = ast.ops().expect("ops walk must yield a top-level Ops"); assert!( diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 30c75806..1d76d550 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -35,6 +35,9 @@ pub mod wmc; mod elixir_decisions; +#[cfg(test)] +#[path = "bash_test_connective_tests.rs"] +mod bash_test_connective_tests; // Gated on the union of every feature any test in the module names — // the `FIXTURES` rows plus the `bash` / `lua` / `c` row set that // `a_language_with_no_member_construct_emits_neither_block` uses. The @@ -44,9 +47,6 @@ mod elixir_decisions; // enabling exactly the languages it exists to cover // (`.claude/rules/testing.md`, #1286). #[cfg(test)] -#[path = "bash_test_connective_tests.rs"] -mod bash_test_connective_tests; -#[cfg(test)] #[cfg(any( feature = "bash", feature = "c", diff --git a/src/metrics/perl_keyword_key_tests.rs b/src/metrics/perl_keyword_key_tests.rs index 0ec5d421..6f66efdd 100644 --- a/src/metrics/perl_keyword_key_tests.rs +++ b/src/metrics/perl_keyword_key_tests.rs @@ -80,8 +80,11 @@ fn measure_with_branches(body: &str) -> ([u64; 4], Halstead) { #[cfg(feature = "perl")] #[test] fn perl_and_key_scores_like_an_or_key() { - // `(or_twin, and_key, decisions)`. - let rows: [(&str, &str, Decisions); 7] = [ + // `(or_twin, and_key, decisions)`. Three rows read the key's *value*: + // the ABC peel steps through an `and` key to it, as it does through a + // `not` key, so a negated value pays its condition. The last is a + // real subtraction before the key, whose `-` still bills. + let rows: [(&str, &str, Decisions); 12] = [ ("my %h; return $h{or};", "my %h; return $h{and};", [2, 0, 0]), ( "my %h; return $h{ or };", @@ -109,6 +112,29 @@ fn perl_and_key_scores_like_an_or_key() { [2, 0, 0], ), ("f(x => 1, or => 2);", "f(x => 1, and => 2);", [2, 0, 0]), + ("f(or => !$a);", "f(and => !$a);", [2, 1, 0]), + ( + "my $y = $c ? 1 : (or => !$a);", + "my $y = $c ? 1 : (and => !$a);", + [3, 3, 1], + ), + ( + "if (or => $x > 1) { return 1; } return 0;", + "if (and => $x > 1) { return 1; } return 0;", + [3, 1, 1], + ), + ( + "my @a = (1 - or => 2);", + "my @a = (1 - and => 2);", + [2, 0, 0], + ), + // The infix `-` is the byte before the first key but signs + // neither: both keys are spelled `and`. + ( + "my @a = ($x - or => 2, or => 3);", + "my @a = ($x - and => 2, and => 3);", + [2, 0, 0], + ), ]; for (twin, key, decisions) in rows { assert_eq!( @@ -194,7 +220,7 @@ fn perl_and_operator_still_scores_like_ampamp() { #[test] fn perl_not_key_scores_like_an_or_key() { // `(or_twin, not_key, decisions, halstead)`. - let rows: [(&str, &str, Decisions, Halstead); 10] = [ + let rows: [(&str, &str, Decisions, Halstead); 13] = [ ( "my %h; return $h{or};", "my %h; return $h{not};", @@ -246,6 +272,29 @@ fn perl_not_key_scores_like_an_or_key() { [3, 2, 1], [10, 11, 6, 6], ), + // A real subtraction before the key: the `-` is infix, no sign, + // and still bills. + ( + "my @a = ($x - or => 2);", + "my @a = ($x - not => 2);", + [2, 0, 0], + [9, 9, 5, 5], + ), + // A comment ending in `-` before the key signs nothing. + ( + "my %h = (\n # -----\n or => 1,\n or => 2,\n);", + "my %h = (\n # -----\n not => 1,\n not => 2,\n);", + [2, 0, 0], + [8, 10, 5, 6], + ), + // A `not` key after an earlier keyword key parses as `not` applied + // to a zero-width bareword the recovery invents. + ( + "f(and => 1, or => 2);", + "f(and => 1, not => 2);", + [2, 0, 0], + [6, 7, 5, 6], + ), ]; for (twin, key, decisions, halstead) in rows { assert_eq!( @@ -316,7 +365,7 @@ fn perl_not_operator_still_scores_like_bang() { /// cognitive], halstead)` rows for /// `perl_dash_key_scores_like_its_quoted_twin`. #[cfg(feature = "perl")] -const DASH_KEY_ROWS: [(&str, &str, [u64; 4], Halstead); 18] = [ +const DASH_KEY_ROWS: [(&str, &str, [u64; 4], Halstead); 20] = [ ( "my %h; return $h{'-foo'};", "my %h; return $h{-foo};", @@ -428,15 +477,31 @@ const DASH_KEY_ROWS: [(&str, &str, [u64; 4], Halstead); 18] = [ [2, 1, 0, 0], [6, 7, 5, 6], ), + // The ABC peel steps through the sign to the key and on to its + // value, so a negated value pays its condition, as its twin's does. + ( + "f('-and' => !$a);", + "f(-and => !$a);", + [2, 1, 1, 0], + [7, 7, 3, 4], + ), + ( + "f('-not' => !$a);", + "f(-not => !$a);", + [2, 1, 1, 0], + [7, 7, 3, 4], + ), ]; -/// Every valid spelling of a `-bareword` key scores exactly as its -/// quoted twin does, in all four metrics (#1545). tree-sitter-perl -/// reads `-foo` as the file test `-f` on a bareword `oo`, `-x => 1` as -/// the file test `-x` swallowing the `=>`, and `-not` as a `-` applied -/// to the keyword. Before the fix each `-foo` billed the operand `oo` -/// and an ABC branch for calling it, and each `-not` / `-and` billed a -/// `-` operator. +/// Each `-bareword` key below scores exactly as its quoted twin does, in +/// all four metrics (#1545). tree-sitter-perl reads `-foo` as the file +/// test `-f` on a bareword `oo`, `-x => 1` as the file test `-x` +/// swallowing the `=>`, and `-not` as a `-` applied to the keyword. +/// Before the fix each `-foo` billed the operand `oo` and an ABC branch +/// for calling it, and each `-not` / `-and` billed a `-` operator. A key +/// whose letter is no file-test letter (`-name`) is not among them: it +/// parses as a `-` applied to a bareword, which still bills the `-` and +/// an ABC branch for the word, as every bareword key does. /// /// The rows pairing a key with a second key equal to the misparsed /// word (`ext`, `not`) or to itself (`-x` twice) pin the operand's @@ -462,6 +527,36 @@ fn perl_dash_key_scores_like_its_quoted_twin() { } } +/// A `-and` / `-not` key keeps its sign when whitespace separates the +/// two: Perl keys `(- not => 1)` as `"-not"`, so it must not share a +/// vocabulary entry with the plain `not` key beside it. The twins' +/// values are `DASH_KEY_ROWS`' for the glued spelling. +#[cfg(feature = "perl")] +#[test] +fn perl_spaced_dash_key_keeps_its_sign() { + for (twin, key) in [ + ("f('-not' => 1, not => 2);", "f(- not => 1, not => 2);"), + ("f('-and' => 1, and => 2);", "f(- and => 1, and => 2);"), + // A comment between the sign and the key leaves it signed. + ( + "f('-not' => 1, not => 2);", + "f(- # c\n not => 1, not => 2);", + ), + ] { + let want = measure_with_branches(twin); + assert_eq!( + want, + ([2, 1, 0, 0], [6, 7, 5, 6]), + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure_with_branches(key), + want, + "`{key}` must score like `{twin}`" + ); + } +} + /// A `-X` that is not a key keeps the reading it had before #1545: a /// real file test, alone, in a condition, on the `_` stat cache, /// stacked, and as a subscript's whole expression, where only a word diff --git a/src/ops.rs b/src/ops.rs index 5dd491f5..137982c5 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -495,6 +495,20 @@ mod tests { ); } + /// The fat comma is a named node, so the generic fallback rendered + /// its kind name, `fat_comma`; it lists as the `=>` written. + #[cfg(feature = "perl")] + #[test] + fn perl_fat_comma_renders_as_written() { + check_ops( + LANG::Perl, + "my %h = (a => 1);", + "foo.pl", + &mut ["()", ";", "=", "=>", "my"], + &mut ["%h", "1", "a"], + ); + } + #[cfg(feature = "python")] #[test] fn python_function_ops() { @@ -1485,15 +1499,14 @@ mod tests { feature = "typescript", ))] fn check_generic_pair(lang: LANG, file: &str, source: &str, operators: &[&str], n1: [u64; 2]) { - let source = Source::new(lang, source.as_bytes()).with_name(Some(file.to_owned())); - let ast = Ast::parse(source).expect("language feature enabled"); + let ast = crate::test_support::parse_named(lang, file, source); let ops = ast.ops().expect("ops walk must yield a top-level Ops"); let mut expected = operators.to_vec(); expected.sort_unstable(); assert_eq!(ops.operators, expected, "{lang:?} vocabulary"); let space = ast - .metrics(crate::MetricsOptions::default()) + .metrics(crate::MetricsOptions::default().with_only(&[crate::Metric::Halstead])) .expect("metrics walk"); let halstead = &space.metrics.halstead; assert_eq!( @@ -1658,6 +1671,89 @@ mod tests { // `argument_list` type form `g(NSArray)` — and never `>`. // `*` four times, `()` and `||` twice; `@end` and `id` bill nothing. // Without the pair, `<` and `>` would each bill six times: [10, 25]. + // expected: the spaced twin `g(a < b, c > d)`, two comparisons. + // Unspaced, tree-sitter-objc reads the arguments as an + // `argument_list`'s `Type<…>` form with an `ERROR` after the `>`, and + // they must keep billing `<` and `>`; the clean `g(NSArray)` + // stays one pair. + #[cfg(feature = "objc")] + #[test] + fn objc_misparsed_comparisons_stay_comparisons_1559() { + const COMPARISONS: &[&str] = &["()", ",", ";", "<", ">", "int", "return", "void", "{}"]; + check_generic_pair( + LANG::Objc, + "foo.m", + "int f(void) { return g(a < b, c > d); }\n", + COMPARISONS, + [9, 10], + ); + check_generic_pair( + LANG::Objc, + "foo.m", + "int f(void) { return g(ad); }\n", + COMPARISONS, + [9, 10], + ); + check_generic_pair( + LANG::Objc, + "foo.m", + "int f(void) { return g(NSArray); }\n", + &["()", "*", ";", "<>", "int", "return", "void", "{}"], + [8, 9], + ); + } + + // expected: the reading before #1559, which billed the two + // comparisons as `<` and `>`. By the C# specification the identifier + // after the `>` makes `F(a < b, c > d)` two arguments, though the + // grammar parses a declaration typed `a`. An `out` declaration + // is a real generic and stays one pair. + #[cfg(feature = "csharp")] + #[test] + fn csharp_misparsed_comparisons_stay_comparisons_1559() { + check_generic_pair( + LANG::Csharp, + "foo.cs", + "class A { void M() { F(a < b, c > d); } }\n", + &["()", ",", ";", "<", ">", "class", "void", "{}"], + [8, 10], + ); + // A member access before the `<` wraps the generic name in a + // `qualified_name`; the comparisons are the same. + check_generic_pair( + LANG::Csharp, + "foo.cs", + "class A { void M() { F(x.a < b, c > d); } }\n", + &["()", ",", ".", ";", "<", ">", "class", "void", "{}"], + [9, 11], + ); + check_generic_pair( + LANG::Csharp, + "foo.cs", + "class A { void M() { F(out List x); } }\n", + &["()", ";", "<>", "class", "int", "out", "void", "{}"], + [8, 10], + ); + // A deconstruction's declarations are `argument`s of a + // `tuple_expression`, not of an argument list: a real generic. + check_generic_pair( + LANG::Csharp, + "foo.cs", + "class A { void M() { (List p, int q) = (1, 2); } }\n", + &["()", ",", ";", "<>", "=", "class", "int", "void", "{}"], + [9, 14], + ); + // A using alias's generic name sits two levels under the root, too + // shallow for the argument-list shape. + check_generic_pair( + LANG::Csharp, + "foo.cs", + "using X = List;\n", + &[";", "<>", "=", "int", "using"], + [5, 5], + ); + } + #[cfg(feature = "objc")] #[test] fn objc_generic_angles_are_one_pair_1559() { diff --git a/src/spaces_tests.rs b/src/spaces_tests.rs index cee8f025..6ff65d68 100644 --- a/src/spaces_tests.rs +++ b/src/spaces_tests.rs @@ -148,11 +148,14 @@ fn mozcpp_bodiless_specifiers_open_no_space_1555() { assert_only_bodied_specifiers_open_spaces::(); } -/// #1555: `is_func_space` and `get_space_kind` agree on every struct / -/// class specifier node, checked node by node because the space tree -/// alone cannot see the getter half — it only labels spaces the checker -/// already opened. The oracle is independent of the `body` field: a -/// specifier defines a type exactly when its text has a `{`. +/// #1555: `promotes_to_func_space_with_code` and +/// `get_space_kind_with_code` — the spellings the walker calls — agree on +/// every struct / class specifier node, checked node by node because the +/// space tree alone cannot see the getter half: it only labels spaces the +/// checker already opened. Today both forward to `is_func_space` / +/// `get_space_kind`, so only a `_with_code` override could split them +/// (grammar-dispatch §7). The oracle is independent of the `body` field: +/// a specifier defines a type exactly when its text has a `{`. #[cfg(any(feature = "cpp", feature = "mozcpp"))] fn assert_specifier_predicates_agree(struct_id: u16, class_id: u16) { use crate::checker::Checker; @@ -180,9 +183,13 @@ fn assert_specifier_predicates_agree(struct_id: u16, class_id: u } else { bodiless += 1; } - assert_eq!(T::Checker::is_func_space(&node), has_body, "{node:?}"); assert_eq!( - T::Getter::get_space_kind(&node), + T::Checker::promotes_to_func_space_with_code(&node, code, Ancestors::unknown()), + has_body, + "{node:?}" + ); + assert_eq!( + T::Getter::get_space_kind_with_code(&node, code, Ancestors::unknown()), if has_body { kind } else { SpaceKind::Unknown }, "{node:?}" ); @@ -2005,8 +2012,6 @@ mod metric_selection_parity { // multi-clause `if` (cognitive, cyclomatic, abc, halstead, tokens), // an early `return` (nexits), parameters (nargs), and a comment // (loc). It is the fixture the non-vacuity assertion below leans on. - // test-lang-gates: hand-written(rust) — a source fixture for - // this grammar, and its text is a string #[cfg(feature = "rust")] const RUST: &str = "\ pub struct Counter { @@ -2035,8 +2040,6 @@ fn choose(a: u32, b: u32) -> u32 { } "; - // test-lang-gates: hand-written(java) — a source fixture for - // this grammar, and its text is a string #[cfg(feature = "java")] const JAVA: &str = "\ public class Shape { @@ -2052,8 +2055,6 @@ public class Shape { } "; - // test-lang-gates: hand-written(python) — a source fixture for - // this grammar, and its text is a string #[cfg(feature = "python")] const PYTHON: &str = "\ class Bag: diff --git a/tests/api/ast_seam_test.rs b/tests/api/ast_seam_test.rs index 02ef30ca..3cd68aa7 100644 --- a/tests/api/ast_seam_test.rs +++ b/tests/api/ast_seam_test.rs @@ -763,14 +763,10 @@ fn preprocess_harvest_feeds_the_macro_masking_pass() { /// failing (`.claude/rules/testing.md`, "Perturb the fixture as well as /// the production line"). Inside, the `proc` is load-bearing for the /// expected sequence. -// test-lang-gates: hand-written(tcl) — a source fixture for this -// grammar, and its text is a string #[cfg(feature = "tcl")] const TCL_SCRIPT_AND_LITERALS: &str = "proc p {x} { puts \"q\" }\nlappend l {a b}\n"; /// The iRules twin, which already had the quoted word inside the body. -// test-lang-gates: hand-written(irules) — a source fixture for this -// grammar, and its text is a string #[cfg(feature = "irules")] const IRULES_SCRIPT_AND_LITERALS: &str = "when HTTP_REQUEST { log local0. \"hi\" }\nlappend l {x y}\n"; diff --git a/utils/check-test-lang-gates-test.py b/utils/check-test-lang-gates-test.py index 1cceddfa..e9e8b637 100755 --- a/utils/check-test-lang-gates-test.py +++ b/utils/check-test-lang-gates-test.py @@ -922,6 +922,149 @@ def test_a_guarded_call_takes_only_its_innermost_gates_languages( self.assertEqual(needs[_named(items, "ts_only").index], {"typescript"}) self.assertEqual(gate.offenders(items, needs), []) + NESTED_ARM = """ +#[cfg(test)] +mod tests { + #[cfg(feature = "python")] + fn shared() {} + + #[cfg(any(feature = "python", feature = "typescript"))] + #[test] + fn nested_arm_case() { + #[cfg(%s)] + { + shared(); + %s + check::(); + #[cfg(feature = "typescript")] + check::(); + } + } +} +""" + + def test_a_call_beside_a_nested_arm_does_not_take_the_arms_languages( + self, + ) -> None: + """The arm sits inside the call's innermost extent, not around it. + + `shared()` is made only where `python` is, so a `python`-gated + helper is right. Unioning the extent's lines handed it the nested + arm's `typescript` too, and reported it stranded on that leg. The + second row is the call made directly in an `any(...)` block, live + on both legs: a `python`-only helper there is a real `E0425`. + """ + self.assertEqual( + _offending(self.NESTED_ARM % ('feature = "python"', "")), + [], + ) + self.assertEqual( + _offending( + self.NESTED_ARM + % ( + 'any(feature = "python", feature = "typescript")', + '#[cfg(feature = "python")]', + ) + ), + [("shared", ["typescript"])], + ) + + GUARD_NAMING_NO_PARSER = """ +#[cfg(test)] +mod tests { + %s + fn shared() {} + + #[test] + fn plain() { + %s + shared(); + } + + #[cfg(feature = "rust")] + #[test] + fn rust_case() { + check_metrics::("fn f() {}", "f.rs", |m| {}); + shared(); + } +} +""" + + def test_a_guard_whose_extent_names_no_language_keeps_the_call( + self, + ) -> None: + """No languages to hand over, so the call stays unconditional. + + `plain` is in every build and calls `shared` wherever its guard + holds, so a `rust`-only `shared` is gated out from under it on + the guard's own leg (`E0425`) -- the gate must refuse that and + accept the ungated helper, as it did before #1562. + """ + for guard in ( + '#[cfg(feature = "python")]', + '#[cfg(feature = "vcs-git")]', + '#[cfg(not(feature = "python"))]', + '#[cfg(any(feature = "python", feature = "typescript"))]', + '#[cfg(all(feature = "python", feature = "vcs-git"))]', + ): + items = _scan(self.GUARD_NAMING_NO_PARSER % ("", guard)) + needs = gate.resolve_needs(items) + self.assertEqual(needs[_named(items, "shared").index], set(), guard) + self.assertEqual(gate.offenders(items, needs), [], guard) + narrow = _scan( + self.GUARD_NAMING_NO_PARSER % ('#[cfg(feature = "rust")]', guard) + ) + self.assertEqual( + [ + (i.name, sorted(w)) + for i, w in gate.over_gated(narrow, gate.resolve_needs(narrow)) + ], + [("shared", ["rust"])], + guard, + ) + + GUARDED_REFERENCES = """ +#[cfg(test)] +mod tests { + #[cfg(any(feature = "python", feature = "typescript"))] + fn shared(_: &[u8]) {} + + #[cfg(%s)] + const ROWS: &[u8] = &[]; + + #[cfg(any(feature = "python", feature = "typescript"))] + #[test] + fn guarded_case() { + #[cfg(feature = "python")] + shared::(ROWS); + #[cfg(feature = "typescript")] + shared::(ROWS); + } +} +""" + + def test_a_table_used_only_under_guarded_calls_takes_their_languages( + self, + ) -> None: + """#1562's `const` half: `C_TYPEDEF_ROWS` beside `assert_operator_rows`. + + Every use of `ROWS` sits under an inner `#[cfg]`. Read as a use by + an always-compiled caller, that cleared its needs, so a gate + missing `typescript` printed OK over an `E0425` on that leg, and + the over-gate check misreported `python` as never used. + """ + source = self.GUARDED_REFERENCES % 'feature = "python"' + self.assertEqual(_offending(source), [("ROWS", ["typescript"])]) + items = _scan(source) + self.assertEqual(gate.over_gated(items, gate.resolve_needs(items)), []) + self.assertEqual( + _offending( + self.GUARDED_REFERENCES + % 'any(feature = "python", feature = "typescript")' + ), + [], + ) + def test_a_cycle_terminates_and_reports_each_narrow_link(self) -> None: """Each half of the cycle is reached from the other's test. diff --git a/utils/check-test-lang-gates.py b/utils/check-test-lang-gates.py index 48182cdc..7008efca 100755 --- a/utils/check-test-lang-gates.py +++ b/utils/check-test-lang-gates.py @@ -826,6 +826,12 @@ class Item: guarded_calls: tuple[tuple[str, frozenset[str]], ...] #: Every identifier in the body, for resolving `const` / `type`. references: frozenset[str] + #: `references` and `guarded_calls` again, for a referent with no + #: call shape (a `const` fixture table): what sits outside every + #: inner `#[cfg(feature = …)]`, and `(name, languages)` per + #: identifier inside one. + unguarded_references: frozenset[str] + guarded_references: tuple[tuple[str, frozenset[str]], ...] parent_index: int | None #: Index into the file's item list; the parent link is by index so #: the dataclass stays comparable and printable. @@ -1073,6 +1079,9 @@ def scan_source( macro_calls = frozenset( m.group(1) for m in CALL_RE.finditer(macro_body) ) + macro_references = frozenset( + m.group(1) for m in REFERENCE_RE.finditer(macro_body) + ) items.append( Item( path=path, @@ -1105,9 +1114,9 @@ def scan_source( calls=macro_calls, unguarded_calls=macro_calls, guarded_calls=(), - references=frozenset( - m.group(1) for m in REFERENCE_RE.finditer(macro_body) - ), + references=macro_references, + unguarded_references=macro_references, + guarded_references=(), parent_index=stack[-1][0] if stack else None, index=len(items), in_test_scope=( @@ -1283,21 +1292,71 @@ def scan_source( # so pairing a call with the outer extent too would hand `a` in # `{ #[cfg(js)] a(); #[cfg(ts)] b(); }` the `ts` its own gate # rightly lacks, and report it stranded on that leg. + # + # The innermost extent's lines still include any arm nested + # inside it, so its set is then filtered to the languages whose + # own build compiles the call: `a` in `#[cfg(js)] { a(); + # #[cfg(ts)] b(); }` keeps `js` and loses `ts`, while a call made + # directly in an `any(js, ts)` block keeps both. An `all(…)` guard + # compiles in no single-language build, so it keeps the whole set. + # + # A guarded line with nothing to hand its callee stays an + # unconditional use, as before #1562: dropping it from both sets + # left the callee gated on its other callers alone, so a + # `#[cfg(feature = "vcs-git")]` or parser-less `python` guard read + # a `rust`-only helper as correct (`E0425` on that leg). A `not(…)` + # gate is false in the all-on build `skipped` reads it against, so + # its languages say nothing about when the call runs. + # + # A `const` table passed to a guarded call is live exactly where + # that call is, so the line's references are attributed the same + # way as its calls. + # + # Every line of one innermost extent gets the same answer, so it is + # computed once per extent; per line, rescanning every extent made + # the gate quadratic in a test module's gated items. guarded_calls: list[tuple[str, frozenset[str]]] = [] + guarded_references: list[tuple[str, frozenset[str]]] = [] + unattributed: set[int] = set() + # Smallest first, so `setdefault` keeps each line's innermost + # extent; a stable sort settles a tie as `min` would, by order. + innermost: dict[int, tuple[int, int]] = {} + for first, last, _ in sorted(extents, key=lambda extent: extent[1] - extent[0]): + for row in range(first, last + 1): + innermost.setdefault(row, (first, last)) + attribution: dict[tuple[int, int], frozenset[str] | None] = {} for n in guarded: - first, last = min( - ((f, l) for f, l, _ in extents if f <= n <= l), - key=lambda span: span[1] - span[0], - ) - languages = frozenset().union( - *(skipped.get(k, ()) for k in range(first, last + 1)) - ) - if languages: + span = innermost[n] + if span not in attribution: + first, last = span + languages = frozenset().union( + *(skipped.get(k, ()) for k in range(first, last + 1)) + ) + + def compiles_alone(feature: str, gates: list[str] = guarded[n]) -> bool: + disabled = KNOWN_FEATURES - enabled_closure(feature) + return all(evaluate_predicate(g, disabled) for g in gates) + + kept = frozenset(filter(compiles_alone, languages)) + if kept or any(map(compiles_alone, KNOWN_FEATURES)): + languages = kept + negated = any( + re.search(r"\bnot\s*\(", p) for f, l, p in extents if (f, l) == span + ) + attribution[span] = languages if languages and not negated else None + languages = attribution[span] + if languages is not None: guarded_calls.extend( (call.group(1), languages) for call in CALL_RE.finditer(lines[n]) ) + guarded_references.extend( + (ref.group(1), languages) for ref in REFERENCE_RE.finditer(lines[n]) + ) + else: + unattributed.add(n) unguarded_body = "\n".join( - "" if n in guarded else lines[n] for n in range(index, body_end + 1) + "" if n in guarded and n not in unattributed else lines[n] + for n in range(index, body_end + 1) ) item = Item( @@ -1328,6 +1387,10 @@ def scan_source( ), guarded_calls=tuple(guarded_calls), references=frozenset(m.group(1) for m in REFERENCE_RE.finditer(body)), + unguarded_references=frozenset( + m.group(1) for m in REFERENCE_RE.finditer(unguarded_body) + ), + guarded_references=tuple(guarded_references), parent_index=parent_index, index=len(items), in_test_scope=in_test_scope, @@ -1648,6 +1711,13 @@ def reachable(item: Item, generic_too: bool) -> set[str]: needs[callee_index] |= { f for f in languages if not gate_excludes(predicate, f) } + # The same for a referent with no call shape: a `const` table + # passed to a guarded call is live exactly where that call is. + for name, languages in item.guarded_references: + index = _resolve_call(items, scopes, item, name) + if index is None or items[index].kind == "fn" or items[index].is_test_fn: + continue + needs[index] |= {f for f in languages if not gate_excludes(predicate, f)} # A referent with even one unconditional caller must stay # unconditional itself, whatever its other callers need. Without @@ -1709,7 +1779,12 @@ def reachable(item: Item, generic_too: bool) -> set[str]: continue # A call behind an inner `#[cfg]` is made only where that # gate holds, so it makes nothing always-compiled (#1562). - if items[index].kind == "fn" and name not in item.unguarded_calls: + unguarded = ( + item.unguarded_calls + if items[index].kind == "fn" + else item.unguarded_references + ) + if name not in unguarded: continue unconditional.add(index) needs[index] = set() From 066daf99b1393fce0d156730b935cc756366d681 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:00:00 -0700 Subject: [PATCH 43/48] perf: find Perl and Kotlin siblings by index The #1545 dash-key workaround found the `=>` after each `-foo` key by scanning the enclosing list from its start, and #1558's Kotlin recovery guard did the same for the token after each parameter modifier. Both are O(width) per node, so a wide argument or parameter list made the walk quadratic: 4 000 Perl keys took 7.45 s and 8 000 Kotlin parameters 2.81 s in a release build. Add Node::children_after and Node::children_before, which seat a cursor on the child with goto_first_child_for_byte (O(log n), as the Tcl slot lookups already do) and fall back to a scan only for a zero-width child. Route the Perl key helpers, the Kotlin guard, and the ERROR-parent operator lookup in Perl ABC's misparse check (the same shape, 4.78 s for 8 000 chains in an unclosed list) through them. All three now scale linearly. Add width probes for the Perl and Kotlin shapes to the scaling gate. Refs #1545 Refs #1558 Also rebaseline `impl Node`'s nom, which the two new methods raise from 36 to 42. --- .bca-baseline.toml | 2 +- big-code-analysis-ast/src/getter/kotlin.rs | 10 +- .../src/lang_helpers/perl.rs | 47 ++++---- big-code-analysis-ast/src/node.rs | 114 ++++++++++++++++++ big-code-analysis-bench/src/shapes.rs | 79 ++++++++++++ docs/development/benchmarking.md | 2 + src/metrics/abc/perl.rs | 65 +++++----- 7 files changed, 255 insertions(+), 64 deletions(-) diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 39400b8d..2980efe2 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -273,7 +273,7 @@ value = 120642.89096069236 path = "big-code-analysis-ast/src/node.rs" qualified = "Node<'a>" metric = "nom" -value = 36.0 +value = 42.0 [[entry]] path = "big-code-analysis-ast/src/parser.rs" diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index e2f4e9e2..65b8765e 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -148,14 +148,14 @@ fn kotlin_modifier_was_written_as_one<'a>(node: &Node<'a>, ancestors: Ancestors< // Whether the first token after `node` is zero-width: one error recovery // inserted where the source has none. +// +// Every parameter's modifier asks, and the climb reaches +// `function_value_parameters`, so the sibling is found without a scan +// from the list's start: that made a long parameter list quadratic. fn kotlin_next_token_is_inserted<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { let mut current = *node; for (parent, _) in ancestors.iter(node) { - if let Some(mut next) = parent - .children() - .skip_while(|child| child.id() != current.id()) - .nth(1) - { + if let Some(mut next) = parent.children_after(¤t).next() { while let Some(first) = next.child(0) { next = first; } diff --git a/big-code-analysis-ast/src/lang_helpers/perl.rs b/big-code-analysis-ast/src/lang_helpers/perl.rs index dec1db62..c278f272 100644 --- a/big-code-analysis-ast/src/lang_helpers/perl.rs +++ b/big-code-analysis-ast/src/lang_helpers/perl.rs @@ -70,15 +70,13 @@ pub fn perl_is_and_key_misparse(expr: &Node) -> bool { /// positions. Remove this, and every call site, once the pin carries the /// upstream fix. #[must_use] -pub fn perl_not_is_key(not: &Node, parent: &Node) -> bool { +pub fn perl_not_is_key<'a>(not: &Node<'a>, parent: &Node<'a>) -> bool { parent.is_error() || parent - .children() + .children_after(not) // The parser marks the `ERROR` it wraps a skipped `=>` in // as an extra, so only comments are passed over here. - .filter(|child| child.is_error() || !child.as_tree_sitter().is_extra()) - .skip_while(|child| child.id() != not.id()) - .nth(1) + .find(|child| child.is_error() || !child.as_tree_sitter().is_extra()) .is_some_and(|next| { (next.is_error() && next.is_child(Perl::FatComma as u16)) || next.start_byte() == next.end_byte() @@ -149,7 +147,7 @@ pub fn perl_is_dash_key_word<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) /// whether the keyword is a key, and a key's sign is part of the string /// whether or not a space separates the two. #[must_use] -pub fn perl_dash_signs_keyword_key(dash: &Node, parent: &Node) -> bool { +pub fn perl_dash_signs_keyword_key<'a>(dash: &Node<'a>, parent: &Node<'a>) -> bool { // A sign is a prefix `-`, whose parent is the subscript's `ERROR` or // a `unary_expression`. An infix `-` is a real subtraction even // before an auto-quoted key (`(1 - and => 2)` subtracts the string), @@ -175,20 +173,17 @@ pub fn perl_dash_signs_keyword_key(dash: &Node, parent: &Node) -> bool { #[must_use] pub fn perl_key_is_signed<'a>(key: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { let mut up = ancestors.iter(key).map(|(ancestor, _)| ancestor); - let Some(parent) = up.next() else { - return false; - }; - let beside = parent - .children() - .filter(|sibling| !sibling.as_tree_sitter().is_extra()) - .take_while(|sibling| sibling.id() != key.id()) - .last(); - if let Some(dash) = beside.filter(|node| node.kind_id() == Perl::DASH as u16) { - return perl_dash_signs_keyword_key(&dash, &parent); - } - up.next().is_some_and(|outer| { - outer.child_by_field_name("operator").is_some_and(|dash| { - dash.kind_id() == Perl::DASH as u16 && perl_dash_signs_keyword_key(&dash, &outer) + up.next().is_some_and(|parent| { + let beside = parent + .children_before(key) + .find(|sibling| !sibling.as_tree_sitter().is_extra()); + if let Some(dash) = beside.filter(|node| node.kind_id() == Perl::DASH as u16) { + return perl_dash_signs_keyword_key(&dash, &parent); + } + up.next().is_some_and(|outer| { + outer.child_by_field_name("operator").is_some_and(|dash| { + dash.kind_id() == Perl::DASH as u16 && perl_dash_signs_keyword_key(&dash, &outer) + }) }) }) } @@ -241,13 +236,13 @@ fn non_extra_count(node: &Node) -> usize { .count() } -// The sibling after `child` in `parent`, comments aside. -fn next_non_extra<'a>(parent: &Node<'a>, child: &Node) -> Option> { +// The sibling after `child` in `parent`, comments aside. Every key of +// a wide `(-a => 1, -b => 2, …)` list asks, so it is found without a +// scan from the list's start (#1545). +fn next_non_extra<'a>(parent: &Node<'a>, child: &Node<'a>) -> Option> { parent - .children() - .filter(|sibling| !sibling.as_tree_sitter().is_extra()) - .skip_while(|sibling| sibling.id() != child.id()) - .nth(1) + .children_after(child) + .find(|sibling| !sibling.as_tree_sitter().is_extra()) } fn first_non_extra<'a>(node: &Node<'a>) -> Option> { diff --git a/big-code-analysis-ast/src/node.rs b/big-code-analysis-ast/src/node.rs index 23e37828..f7ce060b 100644 --- a/big-code-analysis-ast/src/node.rs +++ b/big-code-analysis-ast/src/node.rs @@ -506,6 +506,57 @@ impl<'a> Node<'a> { ChildrenWith { cursor, scan } } + /// The children after `child`, nearest first: what + /// `children().skip_while(|c| c.id() != child.id()).skip(1)` yields, + /// and nothing when `child` is not one of this node's children. + /// + /// That scan costs `child`'s index, so a walk that asks it of every + /// child of a wide list is quadratic in the list (#1545). This finds + /// `child` in `O(log n)` instead; see [`Node::children_before`]. + pub fn children_after(&self, child: &Node<'a>) -> impl Iterator> + use<'a> { + let mut cursor = self.cursor(); + let mut more = + self.seat_on_child(child, &mut cursor).is_some() && cursor.goto_next_sibling(); + std::iter::from_fn(move || { + let next = more.then(|| cursor.node())?; + more = cursor.goto_next_sibling(); + Some(next) + }) + } + + /// The children before `child`, nearest first, and nothing when + /// `child` is not one of this node's children. + /// + /// `child`'s index comes from [`Cursor::goto_first_child_for_byte`] + /// and each sibling from [`Node::child`], both `O(log n)` in the + /// child count, for the reason the first gives: a `repeat()` list is + /// stored as a balanced tree of hidden nodes. + pub fn children_before(&self, child: &Node<'a>) -> impl Iterator> + use<'a> { + let node = *self; + let index = self.seat_on_child(child, &mut self.cursor()).unwrap_or(0); + (0..index).rev().filter_map(move |i| node.child(i)) + } + + // Seats `cursor`, which sits on this node, on `child` and returns its + // index, or `None` when `child` is not one of this node's children. + fn seat_on_child(&self, child: &Node<'a>, cursor: &mut Cursor<'a>) -> Option { + if let Some(index) = cursor.goto_first_child_for_byte(child.start_byte()) + && cursor.node().id() == child.id() + { + return Some(index); + } + // The lookup passes over a zero-width child, which ends at the + // byte it starts at; only error recovery inserts one. + cursor.reset(self); + let mut index = 0; + let mut found = cursor.goto_first_child(); + while found && cursor.node().id() != child.id() { + found = cursor.goto_next_sibling(); + index += 1; + } + found.then_some(index) + } + /// A fresh cursor positioned on this node, for the traversals that /// reuse one cursor across many child scans /// ([`children_with`](Self::children_with)). @@ -1762,6 +1813,69 @@ mod tests { assert!(widest > 2, "fixture must contain a multi-child node"); } + /// `children_after` and `children_before` must yield, for every child + /// of every node, the raw `child(i)` walk on either side of it — and + /// nothing for a node that is not a child. + /// + /// The Kotlin fixture's recovery inserts a zero-width parameter name + /// (`vararg: Int`), which `goto_first_child_for_byte` passes over, so + /// it is the only input that reaches the scan fallback in + /// `seat_on_child`; the count of such children pins that it does. + #[cfg(feature = "kotlin")] + #[test] + fn children_after_and_before_match_the_child_walk() { + let code = b"fun f(vararg: Int, /* c */ b: Int) { g(1, 2, 3) }"; + let tree = Tree::new::(code); + let root = tree.get_root(); + + // Each row is (parent kind, child index, after ids, before ids), and + // a non-child row is (kind, after count, before count) against the + // root, so a failing diff names the node without a message argument. + let mut zero_width = 0; + let (mut observed, mut expected) = (Vec::new(), Vec::new()); + let (mut stray_observed, mut stray_expected) = (Vec::new(), Vec::new()); + for node in root.preorder() { + let raw = node.as_tree_sitter(); + let ids: Vec<_> = (0..raw.child_count() as u32) + .filter_map(|i| raw.child(i)) + .map(|c| c.id()) + .collect(); + for (index, child) in node.children().enumerate() { + zero_width += usize::from(child.start_byte() == child.end_byte()); + observed.push(( + node.kind(), + index, + node.children_after(&child) + .map(|c| c.id()) + .collect::>(), + node.children_before(&child) + .map(|c| c.id()) + .collect::>(), + )); + expected.push(( + node.kind(), + index, + ids[index + 1..].to_vec(), + ids[..index].iter().rev().copied().collect(), + )); + } + if node.id() != root.id() { + stray_observed.push(( + node.kind(), + root.children_after(&node).count(), + root.children_before(&node).count(), + )); + stray_expected.push((node.kind(), 0, 0)); + } + } + assert_eq!(observed, expected); + assert_eq!(stray_observed, stray_expected); + assert!( + zero_width > 0, + "fixture must hold an inserted zero-width child" + ); + } + /// The `O(1)` guard [`Ancestors::checked`] keeps on by default must /// accept every chain a walker really builds — over several grammar /// families, not just the one fixture a failure would surface in. diff --git a/big-code-analysis-bench/src/shapes.rs b/big-code-analysis-bench/src/shapes.rs index 5ea7bc92..dc407d22 100644 --- a/big-code-analysis-bench/src/shapes.rs +++ b/big-code-analysis-bench/src/shapes.rs @@ -487,6 +487,42 @@ pub fn cpp_and_chain(depth: usize) -> String { format!("bool f(bool a){{ return {}a; }}\n", "a && ".repeat(depth)) } +/// Perl: `sub f { return g(-foo => 1, -foo => 1, …); }` — one call +/// whose argument list holds `width` dash-prefixed keys. +/// +/// The width shape for the #1545 dash-key workaround. tree-sitter-perl +/// reads each `-foo` as a file test, and the workaround asks whether a +/// `=>` follows it. Its first form found that `=>` by scanning the list +/// from its start, `O(width)` per key and so `O(width²)` per list: 7.5 s +/// for 4 000 keys in a release build. +#[must_use] +pub fn perl_dash_keys(width: usize) -> String { + format!( + "sub f {{ return g({}); }}\n", + vec!["-foo => 1"; width].join(", ") + ) +} + +/// Kotlin: `inline fun f(noinline p0: () -> Unit, …) {}` — one function +/// with `width` modified parameters. +/// +/// The width shape for #1558's recovery guard, which asks of every +/// parameter modifier whether the next token is an inserted name. Its +/// first form found that token by scanning the parameter list from its +/// start, quadratic in the parameter count. +#[must_use] +pub fn kotlin_noinline_params(width: usize) -> String { + let mut source = String::from("inline fun f("); + for i in 0..width { + if i > 0 { + source.push_str(", "); + } + let _ = write!(source, "noinline p{i:06}: () -> Unit"); + } + source.push_str(") {}\n"); + source +} + /// C++: `int f(int a){ return a + a + … + a; }`. /// /// The shape control for [`cpp_and_chain`]: the same left-nested @@ -1287,6 +1323,49 @@ pub const PROBES: &[Probe] = &[ grows with the width, so a walk that stopped \ counting them is visible in the value column too.", }, + Probe { + name: "halstead/perl-dash-keys", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Halstead], + reading: |m| m.halstead.total_operands(), + }, + render: perl_dash_keys, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1545's dash-key workaround asks of every `-foo` \ + whether a `=>` follows it. Finding that `=>` by \ + scanning the argument list from its start made the \ + walk quadratic in the key count: 1.85 s at 2 000 \ + keys and 7.45 s at 4 000 in a release build, against \ + 0.02 s and 0.03 s once the sibling is found by index. \ + ABC asks through the same helper. The reading is the \ + operand count, which grows with the width.", + }, + Probe { + name: "halstead/kotlin-noinline-params", + lang: LANG::Kotlin, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Halstead], + reading: |m| m.halstead.total_operators(), + }, + render: kotlin_noinline_params, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1558's recovery guard asks of every parameter \ + modifier whether the next token is an inserted name, \ + climbing to the parameter list. Scanning that list \ + from its start made the walk quadratic in the \ + parameter count: 0.73 s at 4 000 parameters and \ + 2.81 s at 8 000 in a release build, against 0.03 s \ + and 0.05 s once the sibling is found by index. The \ + reading is the operator count, which grows with the \ + width.", + }, Probe { name: "halstead/wide-distinct-fn", lang: LANG::Rust, diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index 6b750ee2..a91d314e 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -154,6 +154,8 @@ Read it as follows. | `halstead/wide-distinct-fn` | Rust | width | per-child work at the space-merge boundary (#1106) | linear | | `abc/wide-let-declarators` | JavaScript | width | ABC's structural `const` check per `let` declarator (#1277) | linear | | `loc/wide-cfg-test-mod` | Rust | width | the `exclude_tests` prune arm's row bookkeeping (#1417) | linear | +| `halstead/perl-dash-keys` | Perl | width | the dash-key workaround's `=>` lookup per key (#1545) | linear | +| `halstead/kotlin-noinline-params` | Kotlin | width | the modifier recovery guard's next-token lookup (#1558) | linear | | `cyclomatic/cpp-plus-chain` | C++ | depth | shape control for the three rows below | linear | | `cyclomatic/cpp-and-chain` | C++ | depth | the `&&` / `||` applied-operator gate (#1533) | linear | | `cognitive/cpp-and-chain` | C++ | depth | the boolean-sequence arm on the same chain (#1533) | linear | diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index f87c8804..b4fcc404 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -245,9 +245,12 @@ fn perl_count_slot(slot: Option, conditions: &mut f64) { // a chaining operator, so a block, list or file the climb stops at is // not scanned child by child. Without them every spine node climbed to // the top and every chain scanned its statement list, both quadratic. +// The `ERROR` is the one wide parent left, holding every chain of an +// unclosed list, so the operator before `child` is found by index rather +// than by a scan from the `ERROR`'s start. fn perl_misparse_absorbs_leading_operand<'a>( node: &Node<'a>, - ancestors: impl Iterator>, + mut ancestors: impl Iterator>, ) -> bool { use Perl as P; @@ -257,38 +260,36 @@ fn perl_misparse_absorbs_leading_operand<'a>( return false; } let mut child = *node; - for parent in ancestors { - if perl_leading_slot(&parent).is_some_and(|lead| lead.id() == child.id()) { - child = parent; - continue; + let spine_parent = ancestors.find(|parent| { + let on_spine = perl_leading_slot(parent).is_some_and(|lead| lead.id() == child.id()); + if on_spine { + child = *parent; } - if parent.kind_id() != P::BinaryExpression as u16 && !parent.is_error() { - return false; - } - return parent - .children() - .take_while(|sibling| sibling.id() != child.id()) - .filter(|sibling| !sibling.as_tree_sitter().is_extra()) - .last() - .is_some_and(|operator| { - matches!( - operator.kind_id().into(), - P::LT - | P::GT - | P::LTEQ - | P::GTEQ - | P::Lt - | P::Gt - | P::Le - | P::Ge - | P::EQEQ - | P::BANGEQ - | P::Eq - | P::Ne - ) - }); - } - false + !on_spine + }); + spine_parent.is_some_and(|parent| { + (parent.kind_id() == P::BinaryExpression as u16 || parent.is_error()) + && parent + .children_before(&child) + .find(|sibling| !sibling.as_tree_sitter().is_extra()) + .is_some_and(|operator| { + matches!( + operator.kind_id().into(), + P::LT + | P::GT + | P::LTEQ + | P::GTEQ + | P::Lt + | P::Gt + | P::Le + | P::Ge + | P::EQEQ + | P::BANGEQ + | P::Eq + | P::Ne + ) + }) + }) } // The leading boolean slot of a spine node — a ternary's condition or a From 52868c4cd3bd0d6c57ec6e1207e4619b15afc065 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:00:45 -0700 Subject: [PATCH 44/48] fix(getter/kotlin): skip comments in the name guard #1558's guard bills a modifier keyword only when the token after it is no zero-width name that recovery inserted. A comment between the two is a sibling extra, and the guard took it for that token, so `fun f(vararg /*note*/: Int)` billed a phantom `vararg` operator. Step over extras when finding the next token. A real `vararg` before a commented parameter keeps its operator. Refs #1558 --- big-code-analysis-ast/src/getter/kotlin.rs | 27 ++++++++++++++-------- src/metrics/halstead.rs | 15 ++++++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index 65b8765e..1255c0a1 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -146,24 +146,33 @@ fn kotlin_modifier_was_written_as_one<'a>(node: &Node<'a>, ancestors: Ancestors< host.is_some_and(|host| !host.is_error()) && !kotlin_next_token_is_inserted(node, ancestors) } -// Whether the first token after `node` is zero-width: one error recovery -// inserted where the source has none. +// Whether the first token after `node`, comments aside, is zero-width: +// one error recovery inserted where the source has none. A comment +// between the two is a sibling extra, so `vararg /*c*/: Int` must step +// over it to reach the inserted name, as `vararg: Int` does. A node +// never opens with an extra, which the parser hangs on the level above, +// so the descent takes each first child as it is. // // Every parameter's modifier asks, and the climb reaches // `function_value_parameters`, so the sibling is found without a scan // from the list's start: that made a long parameter list quadratic. fn kotlin_next_token_is_inserted<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { let mut current = *node; - for (parent, _) in ancestors.iter(node) { - if let Some(mut next) = parent.children_after(¤t).next() { + ancestors + .iter(node) + .find_map(|(parent, _)| { + let next = parent + .children_after(¤t) + .find(|sibling| !sibling.as_tree_sitter().is_extra()); + current = parent; + next + }) + .is_some_and(|mut next| { while let Some(first) = next.child(0) { next = first; } - return next.start_byte() == next.end_byte(); - } - current = parent; - } - false + next.start_byte() == next.end_byte() + }) } // `super` (a `super_expression`) names a supertype rather than diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 57d87134..a54cfb58 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -6529,6 +6529,21 @@ end", "fun f(vararg: Int): Int = vararg\n", [4, 5], ), + // The same row with a comment before the inserted name: an + // extra, so the name is still the next token. It scored + // [5, 6], a phantom `vararg` operator. + ( + "parameter named vararg, commented", + "fun f(vararg /*note*/: Int): Int = vararg\n", + [4, 5], + ), + // Control: a real `vararg` before a commented parameter keeps + // its operator. `vararg`, `fun`, `()`, `:`, `{}`. + ( + "vararg modifier, commented", + "fun f(vararg /*note*/ x: Int) {}\n", + [5, 5], + ), // `class`, `()`, `:` ("class parameter named open", "class C(open: Int)\n", [3, 3]), // `val`, `=`, `{}`, `:` From 7bf576d4906e9ee4a18562dea72873ea1ed3adc1 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:10:42 -0700 Subject: [PATCH 45/48] fix(utils): attribute guarded calls by their cfg check-test-lang-gates handed a call behind an inner #[cfg] the parsers named in its block, not the builds its guard admits. A call in an any(python, rust) block naming only PythonParser handed its helper `python` alone, so --fix gated the helper out of the rust build, exited 0, and rejected the correct any(python, rust) gate. Calls were also matched one line at a time, so a turbofish spread over several lines was in neither the guarded nor the unguarded set. Attribute a statement-level guard by evaluating its predicates: the call stays unconditional if they hold with no language enabled, and otherwise hands over every single-language build they admit. A nested item's own #[cfg] is its gate rather than a guard on its container, so those extents keep the parser-named reading. Match calls and references over the whole body, placing each by its start line. The real tree still reports OK with no new markers, and a scratch crate confirms each verdict with rustc (E0425 exactly where the gate reports a stranded build). Refs #1562 --- utils/check-test-lang-gates-test.py | 180 ++++++++++++++++++++++++++-- utils/check-test-lang-gates.py | 126 +++++++++---------- 2 files changed, 237 insertions(+), 69 deletions(-) diff --git a/utils/check-test-lang-gates-test.py b/utils/check-test-lang-gates-test.py index e9e8b637..cb8a2915 100755 --- a/utils/check-test-lang-gates-test.py +++ b/utils/check-test-lang-gates-test.py @@ -922,6 +922,135 @@ def test_a_guarded_call_takes_only_its_innermost_gates_languages( self.assertEqual(needs[_named(items, "ts_only").index], {"typescript"}) self.assertEqual(gate.offenders(items, needs), []) + GUARD_WIDER_THAN_ITS_NAMES = """ +pub struct PythonParser; +#[cfg(test)] +mod tests { + use super::*; + #[cfg(%s)] + fn shared() {} + #[test] + fn guarded_case() { + #[cfg(any(feature = "python", feature = "rust"))] + { + #[cfg(feature = "python")] + let _ = PythonParser; + shared::(); + } + } +} +""" + + def test_a_guarded_call_is_live_wherever_its_guard_holds(self) -> None: + """#1562 review: the block names only `PythonParser`; its guard admits `rust`. + + `shared::()` runs in both builds, so a `python`-only helper is + an `E0425` on the `rust` leg. Read off the parsers in the block, + the gate handed `shared` `python` alone: `--fix` wrote that gate, + exited 0, and the correct `any(python, rust)` was rejected. + """ + self.assertEqual( + _offending(self.GUARD_WIDER_THAN_ITS_NAMES % 'feature = "python"'), + [("shared", ["rust"])], + ) + self.assertEqual( + _offending( + self.GUARD_WIDER_THAN_ITS_NAMES + % 'any(feature = "python", feature = "rust")' + ), + [], + ) + + def test_a_multiline_generic_call_is_attributed_to_its_guard(self) -> None: + """#1562 review: a turbofish spread over lines is still a call. + + Matched one line at a time, `shared::<⏎ PythonParser,⏎ >()` was + in neither the guarded nor the unguarded set, so a `rust`-only + `shared` passed over an `E0425` on the `python` leg. + """ + source = """ +#[cfg(test)] +mod tests { + #[cfg(%s)] + fn shared() {} + #[test] + fn guarded_case() { + #[cfg(feature = "python")] + { + shared::< + PythonParser, + >(); + } + } + #[cfg(feature = "rust")] + #[test] + fn rust_case() { shared::(); } +} +""" + self.assertEqual( + _offending(source % 'feature = "rust"'), [("shared", ["python"])] + ) + self.assertEqual( + _offending(source % 'any(feature = "python", feature = "rust")'), [] + ) + + def test_an_all_guard_hands_over_the_parsers_its_block_names(self) -> None: + """No single-language build admits `all(python, rust)`. + + The call is made only where both are on, so the predicate names + no build to hand over; the parser in the block still says what + the helper needs. Read as unconditional instead, `shared` lost + its needs and the ungated helper passed. + """ + source = """ +#[cfg(test)] +mod tests { + fn shared() {} + #[test] + fn guarded_case() { + #[cfg(all(feature = "python", feature = "rust"))] + { + shared::(); + } + } +} +""" + self.assertEqual(_offending(source), [("shared", ["python"])]) + + def test_a_nested_items_own_gate_is_not_a_guard_on_its_container( + self, + ) -> None: + """A `rust`-gated test inside `mod tests` hands `rust` to nothing. + + The test names no parser, so it needs nothing of `fixtures`; it + is the test's own `#[cfg]`, not a guard on a call the `mod` + makes. Read as a guard, the `mod` handed `fixtures` `rust`, + `fixtures` passed it on to `PYTHON`, and a correctly `python`-gated + table read as stranded on the `rust` leg -- the shape of + `src/spaces_tests.rs`. + """ + source = """ +#[cfg(test)] +mod tests { + #[cfg(feature = "python")] + const PYTHON: &str = ""; + + fn fixtures() -> Vec<(LANG, &'static str)> { + vec![ + #[cfg(feature = "python")] + (LANG::Python, PYTHON), + ] + } + + #[cfg(feature = "rust")] + #[test] + fn rust_case() { + let _ = fixtures(); + } +} +""" + self.assertEqual(_offending(source), []) + NESTED_ARM = """ #[cfg(test)] mod tests { @@ -990,22 +1119,20 @@ def test_a_call_beside_a_nested_arm_does_not_take_the_arms_languages( } """ - def test_a_guard_whose_extent_names_no_language_keeps_the_call( + def test_a_guard_holding_with_no_language_keeps_the_call( self, ) -> None: - """No languages to hand over, so the call stays unconditional. + """A guard true in the no-language build admits the call everywhere. `plain` is in every build and calls `shared` wherever its guard - holds, so a `rust`-only `shared` is gated out from under it on - the guard's own leg (`E0425`) -- the gate must refuse that and - accept the ungated helper, as it did before #1562. + holds, and these hold with every grammar off, so the call stays + unconditional: a `rust`-only `shared` is gated out from under it + (`E0425`) -- the gate must refuse that and accept the ungated + helper, as it did before #1562. """ for guard in ( - '#[cfg(feature = "python")]', '#[cfg(feature = "vcs-git")]', '#[cfg(not(feature = "python"))]', - '#[cfg(any(feature = "python", feature = "typescript"))]', - '#[cfg(all(feature = "python", feature = "vcs-git"))]', ): items = _scan(self.GUARD_NAMING_NO_PARSER % ("", guard)) needs = gate.resolve_needs(items) @@ -1023,6 +1150,43 @@ def test_a_guard_whose_extent_names_no_language_keeps_the_call( guard, ) + def test_a_language_guard_hands_its_builds_without_naming_a_parser( + self, + ) -> None: + """#1562 review: the guard says where the call is made, not the names. + + `plain` names no parser, but its call runs only in the builds its + guard admits, so `shared` needs those beside `rust_case`'s `rust`. + Read off the names, an empty set left the call unconditional and + the helper ungated, dead code on every leg nothing calls it from. + The `any(…)` of exactly those builds is the gate that compiles. + """ + for guard, builds in ( + ('#[cfg(feature = "python")]', ["python"]), + ( + '#[cfg(any(feature = "python", feature = "typescript"))]', + ["python", "typescript"], + ), + ('#[cfg(all(feature = "python", feature = "vcs-git"))]', ["python"]), + ): + items = _scan(self.GUARD_NAMING_NO_PARSER % ("", guard)) + needs = gate.resolve_needs(items) + self.assertEqual( + sorted(needs[_named(items, "shared").index]), sorted(builds + ["rust"]) + ) + narrow = '#[cfg(feature = "rust")]' + self.assertEqual( + _offending(self.GUARD_NAMING_NO_PARSER % (narrow, guard)), + [("shared", builds)], + guard, + ) + exact = "#[cfg(any(%s))]" % ", ".join( + f'feature = "{b}"' for b in sorted(builds + ["rust"]) + ) + self.assertEqual( + _offending(self.GUARD_NAMING_NO_PARSER % (exact, guard)), [], guard + ) + GUARDED_REFERENCES = """ #[cfg(test)] mod tests { diff --git a/utils/check-test-lang-gates.py b/utils/check-test-lang-gates.py index 7008efca..69ef86e2 100755 --- a/utils/check-test-lang-gates.py +++ b/utils/check-test-lang-gates.py @@ -68,6 +68,7 @@ from __future__ import annotations import argparse +import bisect import os import shutil import pathlib @@ -1287,37 +1288,32 @@ def scan_source( # helper then had no derived gate at all, and one missing a # caller's language passed while failing to compile (E0425). # - # Each call is attributed to its *innermost* gate only. An outer - # `#[cfg(any(…))]` block's languages include every nested arm's, - # so pairing a call with the outer extent too would hand `a` in - # `{ #[cfg(js)] a(); #[cfg(ts)] b(); }` the `ts` its own gate - # rightly lacks, and report it stranded on that leg. + # What a guarded call hands its callee is the set of builds its + # guards admit, read off the predicates rather than off the + # parsers named near it. Reading the names made a call in an + # `any(python, rust)` block that names only `PythonParser` hand + # over `python` alone, so `--fix` gated the helper out of the + # `rust` build the call is also made in, and printed OK over the + # `E0425` it caused (#1562 review). The guards are every extent + # around the line, so `a` in `#[cfg(any(js, ts))] { #[cfg(js)] + # a(); }` takes `js` alone, not the outer block's `ts` as well. # - # The innermost extent's lines still include any arm nested - # inside it, so its set is then filtered to the languages whose - # own build compiles the call: `a` in `#[cfg(js)] { a(); - # #[cfg(ts)] b(); }` keeps `js` and loses `ts`, while a call made - # directly in an `any(js, ts)` block keeps both. An `all(…)` guard - # compiles in no single-language build, so it keeps the whole set. - # - # A guarded line with nothing to hand its callee stays an - # unconditional use, as before #1562: dropping it from both sets - # left the callee gated on its other callers alone, so a - # `#[cfg(feature = "vcs-git")]` or parser-less `python` guard read - # a `rust`-only helper as correct (`E0425` on that leg). A `not(…)` - # gate is false in the all-on build `skipped` reads it against, so - # its languages say nothing about when the call runs. + # A guard that holds in the build with no language at all + # (`vcs-git`, `not(python)`) admits the call wherever the caller + # is, so the line stays an unconditional use: dropping it from + # both sets left the callee gated on its other callers alone, and + # `E0425` on that leg. An `all(…)` of languages admits no + # single-language build, so it hands over what its block names, + # as before; with nothing named it too stays unconditional. # # A `const` table passed to a guarded call is live exactly where # that call is, so the line's references are attributed the same # way as its calls. # - # Every line of one innermost extent gets the same answer, so it is - # computed once per extent; per line, rescanning every extent made - # the gate quadratic in a test module's gated items. - guarded_calls: list[tuple[str, frozenset[str]]] = [] - guarded_references: list[tuple[str, frozenset[str]]] = [] - unattributed: set[int] = set() + # Every line of one innermost extent is under the same guards, so + # the answer is computed once per extent; per line, rescanning + # every extent made the gate quadratic in a test module's gated + # items. # Smallest first, so `setdefault` keeps each line's innermost # extent; a stable sort settles a tie as `min` would, by order. innermost: dict[int, tuple[int, int]] = {} @@ -1327,37 +1323,49 @@ def scan_source( attribution: dict[tuple[int, int], frozenset[str] | None] = {} for n in guarded: span = innermost[n] - if span not in attribution: - first, last = span - languages = frozenset().union( - *(skipped.get(k, ()) for k in range(first, last + 1)) - ) - - def compiles_alone(feature: str, gates: list[str] = guarded[n]) -> bool: - disabled = KNOWN_FEATURES - enabled_closure(feature) - return all(evaluate_predicate(g, disabled) for g in gates) + if span in attribution: + continue - kept = frozenset(filter(compiles_alone, languages)) - if kept or any(map(compiles_alone, KNOWN_FEATURES)): - languages = kept - negated = any( - re.search(r"\bnot\s*\(", p) for f, l, p in extents if (f, l) == span - ) - attribution[span] = languages if languages and not negated else None - languages = attribution[span] - if languages is not None: - guarded_calls.extend( - (call.group(1), languages) for call in CALL_RE.finditer(lines[n]) - ) - guarded_references.extend( - (ref.group(1), languages) for ref in REFERENCE_RE.finditer(lines[n]) - ) - else: - unattributed.add(n) - unguarded_body = "\n".join( - "" if n in guarded and n not in unattributed else lines[n] - for n in range(index, body_end + 1) - ) + def admits(disabled: frozenset[str], gates: list[str] = guarded[n]) -> bool: + return all(evaluate_predicate(g, disabled) for g in gates) + + first, last = span + # A nested item's own `#[cfg]` is its gate, not a guard on a + # call this item makes: the nested item derives its own + # needs from its own calls. Read as a guard, a `rust`-gated + # test that names no parser handed `rust` through its `mod` + # to every helper it calls. + if ITEM_RE.match(lines[first]) or admits(KNOWN_FEATURES): + attribution[span] = None + continue + live = frozenset( + f for f in KNOWN_FEATURES if admits(KNOWN_FEATURES - enabled_closure(f)) + ) + named = frozenset().union( + *(skipped.get(k, ()) for k in range(first, last + 1)) + ) + attribution[span] = live or named or None + # Calls are matched over the whole body and placed by where they + # start: matched per line, a call spread over several lines + # (`shared::<\n PythonParser,\n>()`) was in neither set. + line_starts = [0] + for n in range(index, body_end): + line_starts.append(line_starts[-1] + len(lines[n]) + 1) + unguarded_calls: set[str] = set() + unguarded_references: set[str] = set() + guarded_calls: list[tuple[str, frozenset[str]]] = [] + guarded_references: list[tuple[str, frozenset[str]]] = [] + for pattern, unguarded_set, guarded_list in ( + (CALL_RE, unguarded_calls, guarded_calls), + (REFERENCE_RE, unguarded_references, guarded_references), + ): + for hit in pattern.finditer(body): + n = index + bisect.bisect_right(line_starts, hit.start()) - 1 + languages = attribution[innermost[n]] if n in guarded else None + if languages is None: + unguarded_set.add(hit.group(1)) + else: + guarded_list.append((hit.group(1), languages)) item = Item( path=path, @@ -1382,14 +1390,10 @@ def compiles_alone(feature: str, gates: list[str] = guarded[n]) -> bool: hardcoded=frozenset(hardcoded), hand_written=_hand_written_features(raw, attr_line), calls=frozenset(m.group(1) for m in CALL_RE.finditer(body)), - unguarded_calls=frozenset( - m.group(1) for m in CALL_RE.finditer(unguarded_body) - ), + unguarded_calls=frozenset(unguarded_calls), guarded_calls=tuple(guarded_calls), references=frozenset(m.group(1) for m in REFERENCE_RE.finditer(body)), - unguarded_references=frozenset( - m.group(1) for m in REFERENCE_RE.finditer(unguarded_body) - ), + unguarded_references=frozenset(unguarded_references), guarded_references=tuple(guarded_references), parent_index=parent_index, index=len(items), From 65a444f0362f4d88fcd1d5720add84a2a8fdeba3 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:21:35 -0700 Subject: [PATCH 46/48] fix(abc/perl): read separators and parens as Perl does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three ways the Perl ABC walk misread an operand's role: - tree-sitter-perl names `,` and `=>` (`normal_comma`, `fat_comma`), so a trailing separator was taken for a list's last element: `if ($a > 1,)` and `if ($a > 1 =>)` scored 2 conditions against 1 for `if ($a > 1)`, which Deparse reduces them to. No other grammar here names its separator, so the fix is Perl's own operand test. - A pattern's bound/delimiter role was read on its immediate parent, so `$x =~ (/foo/)` and `split((/foo/), $x)` paid the match their bare twins do not, and `$x =~ (s/a/b/)` paid the `=~` that `$x =~ s/a/b/` does not. The role is now read on the outermost `( … )`, stopping at a call's own argument list. Only the binding's right operand is bound: `/foo/ =~ $x` matches `$_` first and now scores 2, as `my $m = /foo/; $m =~ $x` does. - The `split` delimiter exemption ignored the package, so `My::split(/foo/, $x)` scored 0. Only bare `split` and `CORE::split` are the builtin. Refs #1526 Refs #1467 Refs #1540 --- big-code-analysis-bench/src/shapes.rs | 32 ++++++ docs/development/benchmarking.md | 1 + src/metrics/abc.rs | 104 ++++++++++++++++++ src/metrics/abc/perl.rs | 147 +++++++++++++++++++------- 4 files changed, 245 insertions(+), 39 deletions(-) diff --git a/big-code-analysis-bench/src/shapes.rs b/big-code-analysis-bench/src/shapes.rs index dc407d22..4279b756 100644 --- a/big-code-analysis-bench/src/shapes.rs +++ b/big-code-analysis-bench/src/shapes.rs @@ -503,6 +503,17 @@ pub fn perl_dash_keys(width: usize) -> String { ) } +/// Perl: `my @r = (/a/, /a/, …);` — one list of `width` bare matches. +/// +/// The width shape for the Perl ABC pattern-role check, which asks of +/// every pattern whether a `( … )` around it holds it as its value. Its +/// first form read the list's last operand to answer, scanning the whole +/// list per pattern: 7.7 s for 8 000 patterns in a release build. +#[must_use] +pub fn perl_pattern_list(width: usize) -> String { + format!("my @r = ({});\n", vec!["/a/"; width].join(", ")) +} + /// Kotlin: `inline fun f(noinline p0: () -> Unit, …) {}` — one function /// with `width` modified parameters. /// @@ -1344,6 +1355,27 @@ pub const PROBES: &[Probe] = &[ ABC asks through the same helper. The reading is the \ operand count, which grows with the width.", }, + Probe { + name: "abc/perl-pattern-list", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc], + reading: |m| m.abc.conditions_sum(), + }, + render: perl_pattern_list, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "Perl ABC reads a pattern's bound or delimiter role on \ + the outermost `( … )` holding it as its value. \ + Deciding that from the list's last operand scanned \ + the list once per pattern: 1.9 s at 4 000 patterns \ + and 7.7 s at 8 000 in a release build, against \ + 0.015 s and 0.032 s once it looks forward from the \ + pattern. The reading is one condition per bare \ + match, so it grows with the width.", + }, Probe { name: "halstead/kotlin-noinline-params", lang: LANG::Kotlin, diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index a91d314e..50a7e088 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -156,6 +156,7 @@ Read it as follows. | `loc/wide-cfg-test-mod` | Rust | width | the `exclude_tests` prune arm's row bookkeeping (#1417) | linear | | `halstead/perl-dash-keys` | Perl | width | the dash-key workaround's `=>` lookup per key (#1545) | linear | | `halstead/kotlin-noinline-params` | Kotlin | width | the modifier recovery guard's next-token lookup (#1558) | linear | +| `abc/perl-pattern-list` | Perl | width | ABC's pattern-role climb through `( … )` (#1467) | linear | | `cyclomatic/cpp-plus-chain` | C++ | depth | shape control for the three rows below | linear | | `cyclomatic/cpp-and-chain` | C++ | depth | the `&&` / `||` applied-operator gate (#1533) | linear | | `cognitive/cpp-and-chain` | C++ | depth | the boolean-sequence arm on the same chain (#1533) | linear | diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index cbcf2647..f16f646c 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -14790,6 +14790,110 @@ end }); } + // tree-sitter-perl names its separators, so a trailing `,` or `=>` + // was read as the last element of a condition list and paid a slot + // of its own: `if ($a > 1,)` scored 2. `perl -MO=Deparse` reduces + // every `*_sep` row to its `*_twin`, which is where the expected + // values come from; the chain rows put the list inside a `&&` + // operand, which reaches the same peel through + // `perl_count_chain_operands`. + #[cfg(feature = "perl")] + #[test] + fn perl_trailing_separator_scores_like_its_bare_twin() { + check_func_space::( + "sub cmp_twin { my ($a) = @_; if ($a > 1) { g(); } }\n\ + sub cmp_comma { my ($a) = @_; if ($a > 1,) { g(); } }\n\ + sub cmp_fat { my ($a) = @_; if ($a > 1 =>) { g(); } }\n\ + sub var_twin { my ($a) = @_; if ($a) { g(); } }\n\ + sub var_comma { my ($a) = @_; if ($a,) { g(); } }\n\ + sub chain_twin { my ($a, $b) = @_; if (($a > 1) && $b) { g(); } }\n\ + sub chain_comma { my ($a, $b) = @_; if (($a > 1,) && $b) { g(); } }\n", + "foo.pl", + |space| { + assert_members_score( + &space, + &[ + ("cmp_twin", 1, 2), + ("cmp_comma", 1, 2), + ("cmp_fat", 1, 2), + ("var_twin", 1, 2), + ("var_comma", 1, 2), + ("chain_twin", 2, 3), + ("chain_comma", 2, 3), + ], + ); + }, + ); + } + + // A pattern's role is read on the outermost `( … )` around it, which + // is the value inside it, so each `*_paren` row scores as its bare + // `*_twin` (#1467, #1540); `perl -MO=Deparse` prints each pair + // identically. The reversed binding is the other direction: in + // `/foo/ =~ $x` the pattern is the *left* operand, so it first + // matches `$_` and that result is bound (Deparse: `/foo/ =~ /$x/`) — + // the match and the `=~` both pay, as in the explicit `rev_twin`. + #[cfg(feature = "perl")] + #[test] + fn perl_pattern_role_reads_through_parentheses() { + check_func_space::( + "sub bind_twin { my ($x) = @_; return $x =~ /foo/; }\n\ + sub bind_paren { my ($x) = @_; return $x =~ (/foo/); }\n\ + sub bind_parens_m { my ($x) = @_; return $x =~ (((m{foo}))); }\n\ + sub bind_list { my ($x, $y) = @_; return $x =~ (/foo/, $y); }\n\ + sub bind_list_twin { my ($x, $y) = @_; /foo/; return $x =~ $y; }\n\ + sub split_twin { my ($x) = @_; my @p = split(/foo/, $x); return @p; }\n\ + sub split_paren { my ($x) = @_; my @p = split((/foo/), $x); return @p; }\n\ + sub split_sole { my @p = split(/foo/); return @p; }\n\ + sub subst_twin { my ($x) = @_; $x =~ s/a/b/; return $x; }\n\ + sub subst_paren { my ($x) = @_; $x =~ (s/a/b/); return $x; }\n\ + sub rev { my ($x) = @_; return /foo/ =~ $x; }\n\ + sub rev_twin { my ($x) = @_; my $m = /foo/; return $m =~ $x; }\n", + "foo.pl", + |space| { + assert_members_score( + &space, + &[ + ("bind_twin", 1, 1), + ("bind_paren", 1, 1), + ("bind_parens_m", 1, 1), + // A comma operator: `/foo/` matches `$_` and `$y` + // is what binds, so the match pays as in the twin. + ("bind_list", 2, 1), + ("bind_list_twin", 2, 1), + ("split_twin", 0, 1), + ("split_paren", 0, 1), + // The pattern is the whole argument list, which is + // the call's own and no wrapper to climb through. + ("split_sole", 0, 1), + ("subst_twin", 0, 1), + ("subst_paren", 0, 1), + ("rev", 2, 1), + ("rev_twin", 2, 1), + ], + ); + }, + ); + } + + // Only the builtin `split` takes its first pattern as a delimiter: + // `My::split` is a user sub, so the pattern it is handed is an + // ordinary match, as it is for `My::other` (#1467). `CORE::split` + // is the builtin by its full name. + #[cfg(feature = "perl")] + #[test] + fn perl_split_exemption_is_the_builtin_only() { + check_func_space::( + "sub mine { my ($x) = @_; my @p = My::split(/foo/, $x); return @p; }\n\ + sub other { my ($x) = @_; my @p = My::other(/foo/, $x); return @p; }\n\ + sub core { my ($x) = @_; my @p = CORE::split(/foo/, $x); return @p; }\n", + "foo.pl", + |space| { + assert_members_score(&space, &[("mine", 1, 1), ("other", 1, 1), ("core", 0, 1)]); + }, + ); + } + // ---------- Lua ABC tests ---------- #[cfg(feature = "lua")] diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index b4fcc404..0589a4c4 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -10,8 +10,8 @@ )] use super::{ - Abc, Stats, count_boolean_slot, count_each_operand, count_negated_operand, is_operand, - last_operand, peel, wrapped_operand, + Abc, Stats, count_boolean_slot, count_each_operand, count_negated_operand, is_operand, peel, + wrapped_operand, }; use crate::lang_helpers::perl::{ perl_and_is_operator, perl_dash_signs_keyword_key, perl_is_dash_key_word, perl_not_is_key, @@ -51,6 +51,37 @@ use crate::*; // Perl; issue #395). `xor` is no chain: it evaluates both operands, // so like Java's `^` it is a value, and the slot holding it pays // one condition (#1536). + +// Whether `child` can occupy a Perl operand slot. tree-sitter-perl +// names its separators — `normal_comma` (`,`) and `fat_comma` (`=>`) — +// where every other grammar leaves them anonymous, so the shared +// `is_operand` took a trailing one for the list's last element: +// `if ($a > 1,)` and `if ($a > 1 =>)` scored the comma as an unproven +// slot occupant on top of the `>`, where `perl -MO=Deparse` reduces both +// to `if ($a > 1)`. +fn perl_is_operand(child: &Node) -> bool { + is_operand(child) && !matches!(child.kind_id().into(), Perl::NormalComma | Perl::FatComma) +} + +// The last operand of `node`, separators aside: the value a Perl list +// evaluates to in scalar context. +fn perl_last_operand<'a>(node: &Node<'a>) -> Option> { + node.children().filter(perl_is_operand).last() +} + +// `node` with every `( … )` around it taken off. A parenthesised operand +// is the value inside it, so `$x =~ (s/a/b/)` binds the substitution +// as `$x =~ s/a/b/` does. +fn perl_unparenthesized(node: Node<'_>) -> Node<'_> { + let mut node = node; + while node.kind_id() == Perl::Array as u16 + && let Some(inner) = perl_last_operand(&node) + { + node = inner; + } + node +} + // One step of the value peel (see `PeelStep`). // // `Array` is tree-sitter-perl's name for the `(...)` shape used BOTH as @@ -90,17 +121,19 @@ fn perl_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { use Perl as P; match node.kind_id().into() { - P::Array | P::Arguments => last_operand(node).map(|o| (o, false)), + P::Array | P::Arguments => perl_last_operand(node).map(|o| (o, false)), P::ParenthesizedArgument => wrapped_operand(node).map(|o| (o, false)), P::BinaryExpression if node.is_child(P::EQ as u16) => { - last_operand(node).map(|o| (o, false)) + perl_last_operand(node).map(|o| (o, false)) } P::UnaryExpression => { node.child_by_field_name("operator") .and_then(|op| match op.kind_id().into() { - P::Not if perl_not_is_key(&op, node) => last_operand(node).map(|o| (o, false)), + P::Not if perl_not_is_key(&op, node) => { + perl_last_operand(node).map(|o| (o, false)) + } P::And if !perl_and_is_operator(&op, node) => { - last_operand(node).map(|o| (o, false)) + perl_last_operand(node).map(|o| (o, false)) } P::DASH if perl_dash_signs_keyword_key(&op, node) => { wrapped_operand(node).map(|o| (o, false)) @@ -162,7 +195,8 @@ fn perl_condition_scores_itself(expr: &Node) -> bool { // still pays for a rewrite binding through `perl_count_condition`, // because this predicate also keeps `perl_condition_scores_itself` from // claiming it. The grammar names no `left` / `right` field on a -// `binary_expression`, so the right operand is read as its last. +// `binary_expression`, so the right operand is read as its last, with +// any parentheses taken off: `$x =~ (s/a/b/)` is the same edit. // // `$x !~ s///r` and `$x !~ tr///r` parse the same way but are Perl // compile errors ("Using !~ with s///r doesn't make sense"); valid input @@ -173,9 +207,9 @@ fn perl_condition_scores_itself(expr: &Node) -> bool { // no right operand, and reading it per token was quadratic. fn perl_binding_rewrites(binding: &Node) -> bool { binding.kind_id() == Perl::BinaryExpression as u16 - && last_operand(binding).is_some_and(|rhs| { + && perl_last_operand(binding).is_some_and(|rhs| { matches!( - rhs.kind_id().into(), + perl_unparenthesized(rhs).kind_id().into(), Perl::SubstitutionPatternS | Perl::TransliterationTrOrY ) }) @@ -357,7 +391,7 @@ fn perl_count_chain_operands(chain: &Node, leading_absorbed: bool, conditions: & if perl_is_logical_chain(chain) { for operand in chain .children() - .filter(is_operand) + .filter(perl_is_operand) .skip(usize::from(leading_absorbed)) { perl_count_condition(&operand, conditions); @@ -371,52 +405,87 @@ fn perl_count_chain_operands(chain: &Node, leading_absorbed: bool, conditions: & // - **Bound**: the pattern is the right operand of `$x =~ /re/` or // `$x !~ /re/`, whose operator token is already a condition. Read // through the grammar's `operator` field, so `$x ~~ /re/` — a -// smartmatch, which no arm counts — still scores the match once. +// smartmatch, which no arm counts — still scores the match once. Only +// the *right* operand is bound: `/re/ =~ $x` first matches `$_` and +// then binds that result (`perl -MO=Deparse` prints `/re/ =~ /$x/`), +// so it scores the match and the `=~` as its explicit twin +// `my $m = /re/; $m =~ $x` does. // - **`split`'s delimiter**: `split /,/, $s` and `split(/,/, $s)` hand // the pattern to `split` as a separator; nothing is matched against // `$_`, so it is no condition. Only the *first* argument is a // delimiter: a pattern elsewhere in the list is an ordinary match -// whose result `split` receives. The callee is identified by its -// `function_name` bytes (grammar-dispatch §10) — the package -// qualifier is ignored so `CORE::split` is covered too. +// whose result `split` receives. +// +// Either role is read on the pattern's outermost `( … )`, which is the +// value inside it: `$x =~ ((/re/))` binds and `split((/,/), $s)` +// delimits as their bare twins do. A call's own argument list is an +// `Array` too, but it is the call's, not a wrapper around the pattern, +// so the climb stops below one. Whether the pattern is a list's last +// operand is read forward from the pattern, which stops at the next +// operand: read off the list's last operand instead, every pattern of a +// wide `(/a/, /b/, …)` scanned the whole list. fn perl_pattern_is_bound_or_delimiter(pattern: &Node, code: &[u8], ancestors: Ancestors) -> bool { use Perl as P; - let mut climb = ancestors.iter(pattern).map(|(ancestor, _)| ancestor); - climb - .next() - .is_some_and(|parent| match parent.kind_id().into() { - P::BinaryExpression => parent + let mut up = ancestors.iter(pattern).map(|(ancestor, _)| ancestor); + let mut slot = *pattern; + let mut owner = up.next(); + let mut above = up.next(); + while let Some(paren) = owner + && paren.kind_id() == P::Array as u16 + && !paren + .children_after(&slot) + .any(|next| perl_is_operand(&next)) + && !above.is_some_and(perl_is_call_argument_parent) + { + slot = paren; + owner = above; + above = up.next(); + } + owner.is_some_and(|owner| match owner.kind_id().into() { + P::BinaryExpression => { + owner .child_by_field_name("operator") - .is_some_and(|op| matches!(op.kind_id().into(), P::EQTILDE | P::BANGTILDE)), - P::Arguments | P::Array => { - // The first operand, not the first named child: a - // comment is named, so `split( # sep⏎ /,/, $s)` read it - // as the first argument (#1455). - parent - .children() - .find(is_operand) - .is_some_and(|first| first.id() == pattern.id()) - && climb - .next() - .is_some_and(|call| perl_call_is_split(&call, code)) - } - _ => false, - }) + .is_some_and(|op| matches!(op.kind_id().into(), P::EQTILDE | P::BANGTILDE)) + && perl_last_operand(&owner).is_some_and(|rhs| rhs.id() == slot.id()) + } + // The first operand, not the first named child: a comment is + // named, so `split( # sep⏎ /,/, $s)` read it as the first + // argument (#1455). + P::Arguments | P::Array => { + owner + .children() + .find(perl_is_operand) + .is_some_and(|first| first.id() == slot.id()) + && above.is_some_and(|call| perl_call_is_split(&call, code)) + } + _ => false, + }) } -// Whether `call` is a `split` call. Every named child of the two call -// wrappers that take an argument list is the callee or an `args` -// field, so a list whose grandparent is one of them is its arguments. +// Whether `call` is a call of the builtin `split`, spelled bare or as +// `CORE::split`. Every named child of the two call wrappers that take an +// argument list is the callee or an `args` field, so a list whose +// grandparent is one of them is its arguments. The callee's package is +// read as well as its name: `My::split` is a user sub, and the pattern +// it is handed is an ordinary match (grammar-dispatch §10). fn perl_call_is_split(call: &Node, code: &[u8]) -> bool { + let text = |node: &Node| code.get(node.start_byte()..node.end_byte()); matches!( call.kind_id().into(), Perl::CallExpressionWithSpacedArgs | Perl::CallExpressionWithArgsWithBrackets ) && call .children() .find(|child| child.kind_id() == Perl::CallExpressionWithBareword as u16) - .and_then(|callee| callee.child_by_field_name("function_name")) - .is_some_and(|name| code.get(name.start_byte()..name.end_byte()) == Some(b"split")) + .is_some_and(|callee| { + callee + .child_by_field_name("function_name") + .is_some_and(|name| text(&name) == Some(b"split")) + && callee + .children() + .find(|child| child.kind_id() == Perl::PackageName as u16) + .is_none_or(|package| text(&package) == Some(b"CORE")) + }) } fn perl_is_call_argument_parent(parent: Node) -> bool { From becf9c08c6b8cc2cdba8b66aa80a7456f0d9685a Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:30:28 -0700 Subject: [PATCH 47/48] fix(getter/perl): stop a file-test key at a newline #1545 reads a file test that swallowed a `=>` (`(-x => 1)`) as the auto-quoted key `-x`. Perl quotes `-x` there only across spaces and tabs: across a newline, or a comment, which runs to one, it stays the file test on `$_`. `(-f # c\n=> 7)` and `(-f\n=> 7)` both run as `(-f, 7)`, but the grammar recovers the same ERROR around the `=>`, so both billed a fabricated `-f` operand. Require the recovered `=>` to start on the test's own row. The `-foo` and keyword keys keep their reading across a comment or newline, where Perl's result is the same string either way. Refs #1545 --- .../src/lang_helpers/perl.rs | 11 ++++- src/metrics/perl_keyword_key_tests.rs | 48 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/big-code-analysis-ast/src/lang_helpers/perl.rs b/big-code-analysis-ast/src/lang_helpers/perl.rs index c278f272..de72ed6c 100644 --- a/big-code-analysis-ast/src/lang_helpers/perl.rs +++ b/big-code-analysis-ast/src/lang_helpers/perl.rs @@ -102,7 +102,14 @@ pub fn perl_not_is_key<'a>(not: &Node<'a>, parent: &Node<'a>) -> bool { /// call or a string, and stays as the grammar reads it. /// - a key that *is* a file test (`(-x => 1)`) swallows the `=>` and /// the value, wrapping the `=>` in an `ERROR`. A real file test -/// cannot take a `=>` as its operand, so the `ERROR` decides. +/// cannot take a `=>` as its operand, so the `ERROR` decides — when +/// the `=>` is on the test's own line. Perl quotes `-x` only when +/// nothing but spaces and tabs separate it from the `=>`: across a +/// newline or a comment (`(-f # c⏎ => 7)`) it stays the file test on +/// `$_`, the list `(-f, 7)`, and the grammar recovers the same +/// `ERROR` regardless. A comment runs to the end of its line, so the +/// row alone tells the two apart. The `-and` / `-not` and `-foo` +/// keys are quoted across either, as Perl does. /// /// The subscript form of the second shape (`$h{-x}`) recovers into an /// `ERROR` that drops the `-x` token altogether, so nothing is left to @@ -121,7 +128,7 @@ pub(crate) fn perl_file_test_is_dash_key<'a>( .find(|child| child.is_error() || !child.as_tree_sitter().is_extra()) .is_some_and(|operand| { if operand.is_error() { - operand.is_child(Perl::FatComma as u16) + operand.is_child(Perl::FatComma as u16) && operand.start_row() == test.start_row() } else { is_glued_word(&operand, test) && is_key_position(test, ancestors) } diff --git a/src/metrics/perl_keyword_key_tests.rs b/src/metrics/perl_keyword_key_tests.rs index 6f66efdd..e16500af 100644 --- a/src/metrics/perl_keyword_key_tests.rs +++ b/src/metrics/perl_keyword_key_tests.rs @@ -557,6 +557,54 @@ fn perl_spaced_dash_key_keeps_its_sign() { } } +/// Perl quotes a file-test `-X` before `=>` only across spaces and tabs. +/// Across a newline, or a comment, which runs to one, `-f` stays the +/// file test on `$_`: executed, `(-f # c⏎ => 7)` and `(-f⏎ => 7)` both +/// yield `(undef, 7)`, where `(-f \t => 7)` yields `("-f", 7)`. The +/// grammar recovers the same `ERROR` around the `=>` in every spelling, +/// and #1545 billed each as the key `-f`. Each row's twin is the program +/// Perl runs: the comma list for a file test, the quoted key otherwise. +/// A `-foo` key is quoted either way — `-foo` is the string `"-foo"` +/// whether or not the `=>` quotes it — so its commented and +/// line-broken spellings keep their key reading. +#[cfg(feature = "perl")] +#[test] +fn perl_file_test_key_stops_at_a_line_break() { + let comma = ([2, 0, 0, 0], [8, 9, 3, 4]); + let quoted = ([2, 0, 0, 0], [8, 9, 4, 5]); + let word = ([2, 1, 0, 0], [5, 5, 3, 4]); + for (twin, want, key) in [ + ( + "my @p = (-f, 7); return @p;", + comma, + "my @p = (-f # c\n=> 7); return @p;", + ), + ( + "my @p = (-f, 7); return @p;", + comma, + "my @p = (-f\n=> 7); return @p;", + ), + ( + "my @p = ('-f' => 7); return @p;", + quoted, + "my @p = (-f \t => 7); return @p;", + ), + ("f('-foo' => 1);", word, "f(-foo # c\n=> 1);"), + ("f('-foo' => 1);", word, "f(-foo\n=> 1);"), + ] { + assert_eq!( + measure_with_branches(twin), + want, + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure_with_branches(key), + want, + "`{key}` must score like `{twin}`" + ); + } +} + /// A `-X` that is not a key keeps the reading it had before #1545: a /// real file test, alone, in a condition, on the `_` stat cache, /// stacked, and as a subscript's whole expression, where only a word From 8948e9cb1335542256e9ca270279368de07d42ee Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:30:55 -0700 Subject: [PATCH 48/48] docs(changelog): record the external review fixes --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index faeebc38..4f98958b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -370,6 +370,29 @@ derived before these changes. Their re-derivation is tracked in #1569. only to such calls is checked the same way. - **Perl's fat comma lists as `=>` in `bca ops`**, not as the grammar's `fat_comma` node name. Rendering only; no count moves. +- **A trailing Perl `,` or `=>` in a condition adds no condition** + (#1526). `if ($a > 1,)` and `if ($a > 1 =>)` scored 2 against 1 for + `if ($a > 1)`, which Perl reduces them to: the grammar names its + separators, and the trailing one was read as the list's value. +- **A Perl pattern keeps its role inside parentheses** (#1467, #1540). + `$x =~ (/foo/)` and `split((/foo/), $x)` scored the match their bare + forms do not, and `$x =~ (s/a/b/)` scored the `=~` that + `$x =~ s/a/b/` does not. `/foo/ =~ $x`, where the pattern matches + `$_` before its result is bound, now scores 2 rather than 1. +- **Only Perl's builtin `split` exempts its delimiter** (#1467). + `My::split(/foo/, $x)` scored 0; it now scores the match, as + `My::other(/foo/, $x)` does. `CORE::split` is still the builtin. +- **A Kotlin parameter named `vararg` stays a name behind a comment** + (#1558). `fun f(vararg /*note*/: Int)` billed a `vararg` operator + that `fun f(vararg: Int)` does not. +- **A Perl file test before a commented or line-broken `=>` stays a + file test** (#1545). Perl quotes `-f` before `=>` only across spaces + and tabs, so `(-f # c⏎ => 7)` and `(-f⏎ => 7)` run as `(-f, 7)`, but + both billed a `-f` key operand. +- **Wide Perl and Kotlin lists no longer make the walk quadratic** + (#1545, #1558, #1467). A call with thousands of `-foo => …` keys, a + function with thousands of modified parameters, and a list of + thousands of patterns each took seconds; all three are now linear. ## [2.3.0] - 2026-10-07