From 21415ba087a2fa3bb6880ac56d09c81e06721dcd Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:31:53 -0700 Subject: [PATCH 01/34] fix(getter/ts): bill member modifiers as operators TypeScript billed no class-member modifier but `declare`, while C# and Kotlin bill theirs. Each modifier keyword leaf is now an operator in the TS and TSX branch: `public`, `private`, `protected`, `override`, `readonly`, `abstract` and `accessor`. The `accessibility_modifier` / `override_modifier` wrappers stay unlisted, so each bills once. Every word has one kind, so no alias is missed. The leaves bill outside a class as well: parameter properties, `readonly T[]`, a mapped type's `-readonly`, `abstract new () => T`. `static` joins the shared JS-family arm, so JavaScript and MozJS bill it too, in fields, methods and `static {}` blocks. Their one-token `static get` (getter name on the next line) bills as one operator. Used as names (`let readonly = 1`, `obj.static`, `{ public: 1 }`, a field named `static`) the words parse as identifiers and stay operands. The pinned TS grammar misreads a few valid name spellings as the keyword (`abstract();` as a statement, a field `accessor = 1`, an object-type member `abstract: T`); those now bill the operator where they billed nothing, recorded in the getter comment. None occurs in the corpora. Snapshots: 60 pdf.js files move in halstead.* and mi.* only, operand counts unchanged. Submodule 0708c3d9. Fixes #1563 --- .../src/getter/javascript.rs | 7 +- big-code-analysis-ast/src/getter/js_family.rs | 12 ++ big-code-analysis-ast/src/getter/mozjs.rs | 5 +- big-code-analysis-ast/src/getter/tsx.rs | 6 +- .../src/getter/typescript.rs | 21 +- big-code-analysis-book/src/metrics.md | 8 +- src/metrics/halstead.rs | 203 +++++++++++++++++- tests/repositories/big-code-analysis-output | 2 +- 8 files changed, 256 insertions(+), 8 deletions(-) diff --git a/big-code-analysis-ast/src/getter/javascript.rs b/big-code-analysis-ast/src/getter/javascript.rs index c6c1c787..a148fff9 100644 --- a/big-code-analysis-ast/src/getter/javascript.rs +++ b/big-code-analysis-ast/src/getter/javascript.rs @@ -56,9 +56,14 @@ impl Getter for JavascriptCode { // TypeScript, which shares this macro, has no JSX (#1395). // `jsx:` bills element text and its entities as operands, for the // same reason (#1483). + // + // `Staticget` is one token spelling `static get` followed by a + // newline, the static getter whose name starts the next line. It is + // billed beside the shared `static` (#1563) so that form bills a + // modifier rather than nothing; being one node it is one operator. impl_js_family_get_op_type!( Javascript, - op_extras: [OptionalChain, LTSLASH, SLASHGT], + op_extras: [OptionalChain, LTSLASH, SLASHGT, Staticget], operand_extras: [Identifier2, String2], jsx: [JsxText, HtmlCharacterReference], ); diff --git a/big-code-analysis-ast/src/getter/js_family.rs b/big-code-analysis-ast/src/getter/js_family.rs index dc9aea02..4c963d90 100644 --- a/big-code-analysis-ast/src/getter/js_family.rs +++ b/big-code-analysis-ast/src/getter/js_family.rs @@ -24,6 +24,10 @@ // are extras rather than members of the shared arm because // TypeScript is the one grammar of the four with no JSX and // therefore no such variants to name. +// * JavaScript / MozJS: `Staticget`, the one-token `static get` +// these grammars alone have (#1563). +// * TypeScript / TSX: the declaration keywords, type-level operator +// keywords and member modifiers listed at each invocation. // // `$operand_extras` per language: // * JavaScript / MozJS / TSX: `Identifier2`, `String2` — anonymous @@ -237,6 +241,14 @@ macro_rules! impl_js_family_get_op_type { // groups across languages, skewing n1/n2 for accessor-heavy // code (#695). | Set | Get + // `static` is a class-member modifier, billed as C# and + // Kotlin bill theirs (#1563; Java's and Groovy's are + // #1565). It is the bare leaf in a field, a method and + // a `static { … }` block alike, with no modifier + // wrapper, and one kind in all four grammars. Used as a + // name (`obj.static`, `static()`, a field named + // `static`) the grammar emits an identifier instead. + | Static $(| $op_extra)* => TokenRole::Operator, // `Regex` is the literal's own node and contributes one // operand, the way Ruby's `Regex` and Elixir's `Sigil` diff --git a/big-code-analysis-ast/src/getter/mozjs.rs b/big-code-analysis-ast/src/getter/mozjs.rs index 9486d448..b21b6d3a 100644 --- a/big-code-analysis-ast/src/getter/mozjs.rs +++ b/big-code-analysis-ast/src/getter/mozjs.rs @@ -55,10 +55,11 @@ impl Getter for MozjsCode { // `$op_extras` note: they are per-language extras because // TypeScript, which shares this macro, has no JSX (#1395). // `jsx:` bills element text and its entities as operands, for the - // same reason (#1483). + // same reason (#1483). `Staticget`: see the JavaScript invocation + // (#1563). impl_js_family_get_op_type!( Mozjs, - op_extras: [OptionalChain, LTSLASH, SLASHGT], + op_extras: [OptionalChain, LTSLASH, SLASHGT, Staticget], operand_extras: [Identifier2, String2], jsx: [JsxText, HtmlCharacterReference], ); diff --git a/big-code-analysis-ast/src/getter/tsx.rs b/big-code-analysis-ast/src/getter/tsx.rs index d71365b8..5ff4d992 100644 --- a/big-code-analysis-ast/src/getter/tsx.rs +++ b/big-code-analysis-ast/src/getter/tsx.rs @@ -93,6 +93,9 @@ impl Getter for TsxCode { // `Satisfies` / `Keyof` / `Infer` / `Is` / `Asserts2`, the type-level // operator keywords: see the TS invocation (#1561). // + // `Public` through `Accessor`, the member modifiers: see the TS + // invocation (#1563). + // // `generic_angles`: see the TS invocation (#1559). A JSX tag's `<` / // `>` hang from the JSX element nodes, not from a generic list, so // they stay `<` and `>`. @@ -100,7 +103,8 @@ impl Getter for TsxCode { Tsx, op_extras: [ QMARKDOT, PredefinedType, Interface, Enum, Type, Namespace, Module2, Declare, Global, - Satisfies, Keyof, Infer, Is, Asserts2, LTSLASH, SLASHGT, + Satisfies, Keyof, Infer, Is, Asserts2, Public, Private, Protected, Override, Readonly, + Abstract, Accessor, LTSLASH, SLASHGT, ], operand_extras: [Identifier2, String2, TypeIdentifier], predefined_void: PredefinedType, diff --git a/big-code-analysis-ast/src/getter/typescript.rs b/big-code-analysis-ast/src/getter/typescript.rs index 63e458ab..f9a2a596 100644 --- a/big-code-analysis-ast/src/getter/typescript.rs +++ b/big-code-analysis-ast/src/getter/typescript.rs @@ -131,11 +131,30 @@ impl Getter for TypescriptCode { // type assertion wraps a `type_arguments` node, so it is a pair too; // a comparison's `<` / `>` sit under `binary_expression` and stay // their own operators. + // + // #1563 bills the member modifiers, as C# and Kotlin bill theirs: + // `public`, `private`, `protected`, `override`, `readonly`, + // `abstract` and `accessor`, beside the `static` the macro bills for + // the whole family and the `declare` above. Each is its keyword + // leaf, one kind apiece. `public` / `private` / `protected` sit + // under an `accessibility_modifier` and `override` under an + // `override_modifier`; both wrappers stay unlisted, so a modifier + // bills once. The same leaves bill outside a class too: a parameter + // property (`constructor(private x)`), `readonly T[]`, a mapped + // type's `-readonly`, `abstract class` and `abstract new () => T`. + // Used as a name (`let readonly = 1`, `o.public`, `abstract()` in an + // expression, a field named `static`) the grammar emits an + // identifier. The pinned grammar misreads a few valid name + // spellings as the keyword under or beside an ERROR — a statement + // `abstract();`, a field `abstract;` or `accessor = 1`, and an + // object-type member `abstract: T` — so those bill the operator. + // They billed nothing before, and none occurs in the corpora. impl_js_family_get_op_type!( Typescript, op_extras: [ QMARKDOT, PredefinedType, Interface, Enum, Type, Namespace, Module2, Declare, Global, - Satisfies, Keyof, Infer, Is, Asserts2, + Satisfies, Keyof, Infer, Is, Asserts2, Public, Private, Protected, Override, Readonly, + Abstract, Accessor, ], operand_extras: [TypeIdentifier], predefined_void: PredefinedType, diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 19d7c6c5..11568839 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -739,7 +739,13 @@ tokens involved the same way they spell real operators: nothing for either. #1561 made TypeScript's type-level operator keywords operators, as their value-level siblings `as`, `typeof` and `extends` already were: `satisfies`, `keyof`, `infer`, and the - `is` and `asserts` of a type predicate. + `is` and `asserts` of a type predicate. #1563 made TypeScript's + member modifiers operators, as C# and Kotlin bill theirs: `public`, + `private`, `protected`, `override`, `readonly`, `abstract`, + `accessor` and `static`, each the keyword and never its + `accessibility_modifier` or `override_modifier` wrapper. JavaScript + bills `static` too. The same words used as names (`let readonly = 1`, + `obj.static`) stay operands. - **A generic list's angle brackets are one bracket pair.** The `<` and `>` around type arguments and type parameters count as one `<>` operator, billed at the `<` as `()` is at the `(`, and the `>` counts diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index a54cfb58..a636520f 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -605,7 +605,8 @@ mod tests { feature = "csharp", feature = "groovy", feature = "java", - feature = "kotlin" + feature = "kotlin", + feature = "typescript" ))] #[track_caller] fn assert_keywords_are_operands_only( @@ -6791,6 +6792,206 @@ end", assert_operator_rows::("foo.tsx", ROWS); } + /// #1563: TypeScript bills its member modifiers as C# and Kotlin bill + /// theirs, each as the keyword leaf and never as the + /// `accessibility_modifier` / `override_modifier` wrapper. The rows + /// after the class members are the same leaves outside a class; the + /// last two are the words used as names, which add no operator. + #[cfg(feature = "typescript")] + const TS_MEMBER_MODIFIER_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}`, `public`, `private`, `protected`, `=` x3, `;` x3 + ( + "visibility", + "class C {\n public a = 1;\n private b = 2;\n protected c = 3;\n}\n", + [7, 11], + ), + // `class`, `extends`, `{}`, `static`, `readonly`, `override`, + // `=` x2, `;` x2 + ( + "static, readonly, override", + "class D extends B {\n static readonly x = 1;\n override y = 2;\n}\n", + [8, 10], + ), + // `abstract` x2, `class`, `{}`, `()`, `:`, `void`, `;` + ( + "abstract", + "abstract class A {\n abstract m(): void;\n}\n", + [7, 8], + ), + // `class`, `{}`, `accessor`, `=`, `;` + ("accessor", "class E {\n accessor q = 1;\n}\n", [5, 5]), + // `class`, `{}` x3, `static` x2, `()` + ( + "static block and method", + "class F {\n static {}\n static m() {}\n}\n", + [4, 7], + ), + // `class`, `{}` x2, `()`, `private`, `readonly`, `:` x2, `number`, + // `,`, `public`, `string` + ( + "parameter properties", + "class G {\n constructor(private readonly p: number, public o: string) {}\n}\n", + [10, 12], + ), + // `type`, `=`, `readonly`, `string`, `[]`, `;` + ("readonly type", "type R = readonly string[];\n", [6, 6]), + // `type`, `=`, `{}`, `-`, `readonly`, `[]`, `in`, `:`, `;` + ( + "mapped type modifier", + "type M = { -readonly [k in K]: T };\n", + [9, 9], + ), + // `type`, `=`, `abstract`, `new`, `()`, `=>`, `void`, `;` + ( + "abstract constructor type", + "type A = abstract new () => void;\n", + [8, 8], + ), + // `let`, `const` x2, `=` x3, `;` x4, `.`, `()`, `{}`, `:` x5, + // `,` x4 — no modifier + ( + "names", + "let readonly = 1;\nobj.static;\nconst a = abstract();\n\ + const o = { public: 1, private: 2, protected: 3, override: 4, accessor: 5 };\n", + [9, 22], + ), + // `class`, `{}` x2, `static`, `=`, `;` x2, `()` — the modifier + // bills once; the field named `static`, the method named + // `readonly` and the field named `public` bill none + ( + "member names", + "class K {\n static static = 1;\n readonly() {}\n public;\n}\n", + [6, 8], + ), + ]; + + #[cfg(feature = "typescript")] + #[test] + fn typescript_member_modifiers_are_operators_1563() { + assert_operator_rows::("foo.ts", TS_MEMBER_MODIFIER_ROWS); + assert_operator_rows::("foo.tsx", TS_MEMBER_MODIFIER_ROWS); + } + + /// #1563: the issue's own fixture, by vocabulary. Every modifier + /// reaches the operator list once, by its word and never by its + /// wrapper's kind name, and the names stay the only operands. + #[cfg(feature = "typescript")] + #[test] + fn typescript_member_modifier_vocabulary_1563() { + const SOURCE: &str = "abstract class C {\n declare x: number;\n \ + public static readonly y = 1;\n private z = 2;\n \ + protected override w = 3;\n abstract m(): void;\n \ + accessor q = 4;\n}\n"; + const OPERATORS: &[&str] = &[ + "()", + ":", + ";", + "=", + "abstract", + "accessor", + "class", + "declare", + "number", + "override", + "private", + "protected", + "public", + "readonly", + "static", + "void", + "{}", + ]; + const OPERANDS: &[&str] = &["1", "2", "3", "4", "C", "m", "q", "w", "x", "y", "z"]; + for (file, ops) in [ + ("foo.ts", ops_of::(SOURCE, "foo.ts")), + ("foo.tsx", ops_of::(SOURCE, "foo.tsx")), + ] { + assert_eq!(ops.operators, OPERATORS, "{file}"); + assert_eq!(ops.operands, OPERANDS, "{file}"); + } + } + + /// #1563: the modifier words used as names are operands and never + /// operators, in an expression, a member access, a call, an object + /// key and a class member name. No getter edit makes this fail: the + /// modifier arms key on the keyword kinds, and the grammar emits an + /// identifier for each name. It pins that grammar property, so a + /// grammar bump or a text-keyed arm that bills a name trips it. + #[cfg(feature = "typescript")] + #[test] + fn typescript_modifier_words_as_names_are_operands_1563() { + const SOURCE: &str = "let readonly = 1;\nobj.static;\nconst a = abstract();\n\ + const o = { public: 1, private: 2, protected: 3, override: 4, \ + accessor: 5 };\nclass K { static() {} }\n"; + const KEYWORDS: &[&str] = &[ + "readonly", + "static", + "abstract", + "public", + "private", + "protected", + "override", + "accessor", + ]; + assert_keywords_are_operands_only::(SOURCE, "foo.ts", KEYWORDS); + assert_keywords_are_operands_only::(SOURCE, "foo.tsx", KEYWORDS); + } + + /// #1563: JavaScript's `static` bills as TypeScript's does, in a + /// field, a method and a `static { … }` block. `static get` with the + /// name on the next line is one token, so it is one operator. The + /// last row is the word used as names, which adds no operator. + #[cfg(any(feature = "javascript", feature = "mozjs"))] + const JS_STATIC_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}`, `static`, `=`, `;` + ("static field", "class K {\n static x = 1;\n}\n", [5, 5]), + // `class`, `{}` x3, `static` x2, `()` + ( + "static method and block", + "class K {\n static m() {}\n static {}\n}\n", + [4, 7], + ), + // `class`, `{}` x2, `static`, `get`, `()` + ( + "static getter", + "class K {\n static get y() {}\n}\n", + [5, 6], + ), + // `class`, `{}` x2, `static get`, `()` + ( + "static get before a newline", + "class K {\n static get\n y() {}\n}\n", + [4, 5], + ), + // `.`, `;` x3, `()`, `var`, `=`, `{}`, `:` — no `static` + ( + "names", + "obj.static;\nstatic();\nvar o = { static: 1 };\n", + [7, 9], + ), + ]; + + #[cfg(any(feature = "javascript", feature = "mozjs"))] + #[test] + fn javascript_static_is_an_operator_1563() { + const NAMES: &str = "obj.static;\nstatic();\nvar o = { static: 1 };\n\ + class K { static static = 1; }\n"; + #[cfg(feature = "javascript")] + { + assert_operator_rows::("foo.js", JS_STATIC_ROWS); + let ops = ops_of::(NAMES, "foo.js"); + assert!(ops.operands.iter().any(|o| o == "static"), "{ops:?}"); + assert!(ops.operators.iter().any(|o| o == "static"), "{ops:?}"); + } + #[cfg(feature = "mozjs")] + { + assert_operator_rows::("foo.js", JS_STATIC_ROWS); + let ops = ops_of::(NAMES, "foo.js"); + assert!(ops.operands.iter().any(|o| o == "static"), "{ops:?}"); + assert!(ops.operators.iter().any(|o| o == "static"), "{ops:?}"); + } + } + #[cfg(feature = "bash")] #[test] fn bash_operators_and_operands() { diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index a737c2ff..0708c3d9 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit a737c2ff02d8b9884700bf5e6d024c10e14ab9d1 +Subproject commit 0708c3d941e278d1c304fd981cfc3f8b7752fe22 From d3f058a8ed2e046de31433d0001eee18bdfed619 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:44:23 -0700 Subject: [PATCH 02/34] fix(getter/kotlin): bill six declaration keywords Kotlin billed none of `typealias`, `init`, `where`, `by`, `get` or `set`, while it bills its other declaration keywords and every modifier. Each is now an operator, as C# bills `get` / `set` / `init` / `where` and TypeScript `type`. Each is the keyword leaf; the `type_alias`, `anonymous_initializer`, `type_constraints`, `property_delegate` / `explicit_delegation`, `getter` and `setter` wrappers stay unlisted, so each bills once. Every word has one kind. `get` and `set` bill only directly under `getter` / `setter`: the same leaves are the `@get:` / `@set:` use-site targets, which bill nothing like `@field:`, and kotlin-ng strands a `get(1)` call after a local `val f = 6` in an ERROR. `by`, `where` and `init` used as names parse as identifiers, and `typealias` is a hard keyword, so those four need no guard. The accessor body's `field` is an identifier and stays an operand. No corpus snapshot moves: the one Kotlin corpus file uses none of the words. Fixes #1564 --- big-code-analysis-ast/src/getter/kotlin.rs | 34 +++++++- big-code-analysis-book/src/metrics.md | 7 +- src/metrics/halstead.rs | 98 ++++++++++++++++++++++ 3 files changed, 137 insertions(+), 2 deletions(-) diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index 1255c0a1..83571b56 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -340,6 +340,31 @@ impl Getter for KotlinCode { }; } + // The accessor keywords, billed as C# bills its `get` / `set` + // (#1564), but only directly under their `getter` / `setter`, + // which stay unlisted. The same leaves are the `@get:` / `@set:` + // use-site targets, which bill nothing, like `@field:` and + // `@property:`; and where kotlin-ng misreads a call `get(1)` after + // a local `val f = 6`, an `ERROR` holds the leaf. Used as a name + // otherwise (`val get = 1`, `x.set(v)`, `fun get()`) the word + // parses as an identifier. The grammar also reads a local + // `val f = 6` followed by `set = 2` as a setter, which bills + // `set`. `field` in an accessor body is an identifier, so it is an + // operand like any other variable. An early return for the same + // reason as the modifiers'. + let accessor = match kind { + Get => Some(Kotlin::Getter), + Set => Some(Setter), + _ => None, + }; + if let Some(accessor) = accessor { + return if ancestors.parent_has_kind(node, accessor as u16) { + TokenRole::Operator + } else { + TokenRole::Unknown + }; + } + match kind { // Operator: control flow keywords If | Else | When | For | While | Do | Try | Catch | Finally | Throw | Return @@ -348,8 +373,15 @@ impl Getter for KotlinCode { // keyword leaf beside `class` and `object` (#1556); its // `class_declaration` wrapper stays unlisted. `companion` is // the one modifier billed here; see the early return above. + // `typealias`, `init`, `where` and `by` bill as C# bills + // `where` and TypeScript `type` (#1564), each the leaf of its + // `type_alias` / `anonymous_initializer` / `type_constraints` + // / `property_delegate` or `explicit_delegation` wrapper. + // `typealias` is a hard keyword; the other three, used as + // names (`val by = 1`, `where(x)`, `init.f()`), parse as + // identifiers, so the leaf exists only in its construct. | Class | Interface | Fun | Object | Val | Var | In | Is | As | AsQMARK | BANGis - | BANGin | Constructor | Companion + | BANGin | Constructor | Companion | Typealias | Init | Where | By // Operator: brackets, separators, terminators | SEMI | COMMA | COLONCOLON | DOT | LBRACE | LBRACK | LPAREN // Operator: assignment and arithmetic diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 11568839..129ec919 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -745,7 +745,12 @@ tokens involved the same way they spell real operators: `accessor` and `static`, each the keyword and never its `accessibility_modifier` or `override_modifier` wrapper. JavaScript bills `static` too. The same words used as names (`let readonly = 1`, - `obj.static`) stay operands. + `obj.static`) stay operands. #1564 made Kotlin's `typealias`, `init`, + `where`, `by`, `get` and `set` operators, as C# bills its `get`, + `set`, `init` and `where`. The `@get:` and `@set:` annotation targets + bill nothing, like `@field:`. The backing `field` in an accessor is an + operand, and the words used as names (`val by = 1`, `x.get(i)`) stay + operands. - **A generic list's angle brackets are one bracket pair.** The `<` and `>` around type arguments and type parameters count as one `<>` operator, billed at the `<` as `()` is at the `(`, and the `>` counts diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index a636520f..49ea2721 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -6588,6 +6588,104 @@ end", ); } + /// #1564: `typealias`, `init`, `where`, `by`, `get` and `set` bill + /// as Kotlin's other declaration keywords do, and as C# bills its + /// `get` / `set` / `init` / `where`. Each is the keyword leaf, never + /// its wrapper. The rows after "accessors" are controls: the + /// `@get:` / `@set:` use-site targets and the words used as names + /// add no operator. + #[cfg(feature = "kotlin")] + #[test] + fn kotlin_declaration_keywords_are_operators_1564() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `typealias`, `=` + ("typealias", "typealias A = Int\n", [2, 2]), + // `class`, `init`, `{}` x2 + ("init", "class C {\n init {}\n}\n", [3, 4]), + // `class`, `<>`, `where`, `:` + ("class where", "class C where T : Any\n", [4, 4]), + // `fun`, `<>`, `()`, `where`, `:`, `{}` + ("function where", "fun f() where T : Any {}\n", [6, 6]), + // `val`, `by`, `{}` + ("property delegate", "val d by lazy { 1 }\n", [3, 3]), + // `class`, `()`, `:` x2, `by` + ("class delegation", "class D(d: I) : I by d\n", [4, 5]), + // `var`, `:`, `=` x3, `get`, `()` x2, `set`, `{}` + ( + "accessors", + "var p: Int = 0\n get() = field\n set(v) { field = v }\n", + [7, 10], + ), + // `var`, `=`, `:` x2: the use-site targets bill nothing, as + // `@field:` does + ("use-site targets", "@get:A @set:B var q = 0\n", [3, 4]), + // `val`, `=` + ( + "names", + "val get = 1\nval set = 2\nval by = 3\nval where = 4\nval init = 5\n", + [2, 10], + ), + // `fun`, `()` x4, `:`, `{}`, `.` x2 + ( + "calls", + "fun g(x: X) {\n x.get(by)\n x.set(where)\n init(x)\n}\n", + [5, 9], + ), + // `fun`, `()` x2, `{}`, `val`, `=`: valid Kotlin that kotlin-ng + // misreads, leaving the call's `get` leaf in an `ERROR` + ( + "call after a local val", + "fun g() {\n val f = 6\n get(1)\n}\n", + [5, 6], + ), + ]; + assert_operator_rows::("foo.kt", ROWS); + } + + /// #1564: the six words are listed by their own spelling, and the + /// accessor body's `field` stays an operand like any variable. + #[cfg(feature = "kotlin")] + #[test] + fn kotlin_declaration_keyword_vocabulary_1564() { + let ops = ops_of::( + "typealias A = Int\nclass C where T : Any {\n init {}\n \ + val d by lazy { 1 }\n var p: Int = 0\n get() = field\n \ + set(v) { field = v }\n}\n", + "foo.kt", + ); + for word in ["typealias", "init", "where", "by", "get", "set"] { + assert!( + ops.operators.iter().any(|op| op == word), + "{word}: {:?}", + ops.operators + ); + } + assert!( + ops.operands.iter().any(|op| op == "field"), + "{:?}", + ops.operands + ); + + // The words used as names are operands, not merely non-operators: + // the operator rows above cannot see a name that bills nothing. + let names = ops_of::( + "fun get(x: X) {\n val set = 1\n val by = 2\n x.init(where(set, by))\n}\n", + "foo.kt", + ); + for word in ["init", "where", "by", "get", "set"] { + assert!( + names.operands.iter().any(|op| op == word), + "{word}: {:?}", + names.operands + ); + assert!( + !names.operators.iter().any(|op| op == word), + "{word}: {:?}", + names.operators + ); + } + } + /// #1557 rows shared by the two C++ clones: the declaration keywords /// #1552 left out, one per row, plus the two disambiguator positions /// (`typename T::x`, `x.template f`) that bill the same leaf. From 5fd5d9b8bc74f02dc95e885c552086f0c88aedfb Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:05:48 -0700 Subject: [PATCH 03/34] fix(getter/java): bill every Java and Groovy modifier Java and Groovy billed `abstract`, `final`, `transient`, `synchronized` and `default` but no other modifier, while C# and Kotlin bill all of theirs. Each modifier keyword leaf is now an operator in both: `public`, `private`, `protected`, `static`, `native`, `strictfp`, `volatile`, `sealed` and `non-sealed`, plus the `permits` clause keyword beside the already-billed `extends` and `implements`. Java's `modifiers` wrapper stays unlisted and Groovy has none, so each modifier bills once; an annotation inside `modifiers` is unchanged. No word has a numeric-suffix alias. Java's `default` is one kind for the switch label and the interface-method modifier and already billed. The `static` of `import static` and Java's `requires static` bills nothing, as `import` and `requires` bill nothing. Java parses `sealed` and `permits` used as names as identifiers, so they stay operands. Groovy fails a statement that opens with one (`sealed = 1`) into an ERROR holding the keyword, so its `sealed` and `permits` bill only under their declaration; the gate joins the existing `trait` / `record` one in a single (keyword, owner) table. No corpus snapshot moves: the DeepSpeech corpus globs only C++, and no Java or Groovy corpus is snapshotted. Fixes #1565 --- .bca-baseline.toml | 6 + big-code-analysis-ast/src/getter/groovy.rs | 60 +++- big-code-analysis-ast/src/getter/java.rs | 23 ++ big-code-analysis-book/src/metrics.md | 12 +- src/metrics/halstead.rs | 316 ++++++++++++++++++--- src/ops.rs | 8 +- 6 files changed, 366 insertions(+), 59 deletions(-) diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 2980efe2..165af7df 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -245,6 +245,12 @@ qualified = "IrulesCode::get_op_type" metric = "halstead.effort" value = 51477.285144213616 +[[entry]] +path = "big-code-analysis-ast/src/getter/java.rs" +qualified = "JavaCode::get_op_type" +metric = "halstead.effort" +value = 50584.467250345944 + [[entry]] path = "big-code-analysis-ast/src/getter/perl.rs" qualified = "PerlCode::get_op_type" diff --git a/big-code-analysis-ast/src/getter/groovy.rs b/big-code-analysis-ast/src/getter/groovy.rs index a93d2a35..115e739e 100644 --- a/big-code-analysis-ast/src/getter/groovy.rs +++ b/big-code-analysis-ast/src/getter/groovy.rs @@ -20,6 +20,34 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { is_misparse: never_misparsed, }; +/// Whether a contextual keyword leaf (`trait`, `record`, `sealed`, +/// `permits`) sits directly under the construct it introduces. +/// +/// Valid Groovy may use any of the four as a name. Inside a class the +/// grammar parses that name as an `identifier`, but a script-level +/// statement that opens with one (`record = 3`, `String record`, +/// `void record() {}`, `sealed.call()`) fails into an `ERROR` holding +/// the keyword leaf. Billing the leaf there would make a variable an +/// operator (#1552, #1565). The pinned grammar also misreads a *nested* +/// `sealed class` as a field typed `sealed`, so that `sealed` bills as +/// a type-name operand. +fn contextual_keyword_owns<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + use Groovy::*; + + ancestors.parent(node).is_some_and(|parent| { + matches!( + (node.kind_id().into(), parent.kind_id().into()), + (Trait, TraitDeclaration) + | (Record, RecordDeclaration) + | (Permits, PermitsClause) + | ( + Sealed, + ClassDeclaration | InterfaceDeclaration | TraitDeclaration + ) + ) + }) +} + impl Getter for GroovyCode { /// Names the space, synthesising one for constructs that carry no /// name token (#1184). @@ -103,9 +131,11 @@ impl Getter for GroovyCode { if GENERIC_ANGLES.is_closer(node, ancestors) { return TokenRole::Unknown; } - // Mirrors `JavaCode`'s minimal classification — modifiers - // (`Public`, `Static`, …) and module keywords (`Package`, - // `Import`, …) are excluded. The type-declaration keywords + // Mirrors `JavaCode`'s classification: modifiers are billed + // (#1565) and module keywords (`Package`, `Import`, …) are + // excluded. The grammar has no `modifiers` wrapper — each + // modifier leaf sits directly under its declaration — so each + // bills once. The type-declaration keywords // (`class`, `interface`, `trait`, `enum`, `record`) are billed, // as Java's are since #1552: no `*Declaration` wrapper is // classified, so the keyword leaf is the only node that can @@ -178,18 +208,17 @@ impl Getter for GroovyCode { Super if ancestors.parent_has_kind(node, Wildcard as u16) => TokenRole::Operator, Super => TokenRole::Operand, - // `trait` and `record` are contextual keywords, so valid - // Groovy may use either as a name. Inside a class the grammar - // parses that name as an `identifier`, but at script level it - // fails `record = 3`, `String record` and `void record() {}` - // into an `ERROR` holding the keyword leaf. Billing the leaf - // there would make a variable an operator, so it bills only - // under the declaration it introduces (#1552). - Trait if ancestors.parent_has_kind(node, TraitDeclaration as u16) => { + // `trait`, `record`, `sealed` and `permits` are contextual + // keywords, so each bills only under the construct it + // introduces (`contextual_keyword_owns`); elsewhere it falls + // through to `Unknown`. + Trait | Record | Sealed | Permits if contextual_keyword_owns(node, ancestors) => { TokenRole::Operator } - Record if ancestors.parent_has_kind(node, RecordDeclaration as u16) => { - TokenRole::Operator + // `import static`'s `static` bills nothing, as `import` does + // not; Java makes the same call. + Static if ancestors.parent_has_kind(node, ImportDeclaration as u16) => { + TokenRole::Unknown } // Control-flow + keyword operators (mirrors Java's set, @@ -198,6 +227,11 @@ impl Getter for GroovyCode { If | Else | Switch | Case | Try | Catch | Throw | Throws | For | While | Continue | Break | Do | Finally | New | Return | Default | Abstract | Assert | Instanceof | Extends | Final | Implements | Transient | Synchronized | Def | In | As + // The rest of the modifiers (#1565). `non-sealed` cannot be + // spelled as a name; `non-sealed` as a subtraction parses as + // `non`, `-` and an identifier `sealed`. + | Public | Private | Protected | Static | Native | Strictfp | Volatile + | NonDASHsealed // Type-declaration keywords (#1552). Unlike Java's, `class` // needs no class-literal guard: `String.class` parses as a // `field_access` whose `class` is a plain `identifier`. diff --git a/big-code-analysis-ast/src/getter/java.rs b/big-code-analysis-ast/src/getter/java.rs index b5439162..073ba2d5 100644 --- a/big-code-analysis-ast/src/getter/java.rs +++ b/big-code-analysis-ast/src/getter/java.rs @@ -116,12 +116,35 @@ impl Getter for JavaCode { { TokenRole::Unknown } + // `static` is a modifier (below, #1565) except in an import + // or module directive (`import static`, `requires static`), + // whose own keyword — `import`, `requires` — bills nothing + // here, so neither does the word it qualifies. + Static + if ancestors.parent(node).is_some_and(|p| { + matches!(p.kind_id().into(), ImportDeclaration | RequiresModifier) + }) => + { + TokenRole::Unknown + } // Operator: control flow | If | Else | Switch | Case | Try | Catch | Throw | Throws | Throws2 | For | While | Continue | Break | Do | Finally // Operator: keywords | New | Return | Default | Abstract | Assert | Instanceof | Extends | Final | Implements | Transient | Synchronized | VoidType + // Operator: the rest of the modifiers, billed as C# and Kotlin + // bill theirs (#1565). Each is its keyword leaf, one kind + // apiece; the `modifiers` wrapper stays unlisted, so a + // modifier bills once and an annotation inside it adds only + // its own tokens. `default` above is one kind for a `switch` + // label and an interface method's modifier. `sealed` and + // `permits` (`Permits2`; `Permits` is the clause) are + // contextual, and the grammar parses each used as a name + // (`var sealed = 1`, `int permits`) as an `identifier`, so + // they need no guard; `non-sealed` cannot be a name. + | Public | Private | Protected | Static | Native | Strictfp | Volatile | Sealed + | NonDASHsealed | Permits2 // Operator: type-declaration keywords, billed as C# bills // its own (#1552). Only the leaves: no `*_declaration` // wrapper is listed, so a declaration bills its keyword once. diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 129ec919..fc04b2b8 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -717,8 +717,7 @@ tokens involved the same way they spell real operators: `interface`, `record`, `trait`, `@interface`, `@implementation` and `@protocol`. Java's `C.class` is not billed: it stays one operand. Kotlin's `interface` followed in #1556, and since #1558 Kotlin bills - every modifier, as C# does (Java and Groovy bill `abstract` and - `final` but no visibility keyword or `static`, #1565): the `enum` of + every modifier, as C# does: the `enum` of `enum class`, `data`, `sealed`, the visibility keywords, `open`, `override`, `suspend`, `reified`, `out` and the rest each count once, as the keyword and never as its `modifiers` wrapper. The same words @@ -750,7 +749,14 @@ tokens involved the same way they spell real operators: `set`, `init` and `where`. The `@get:` and `@set:` annotation targets bill nothing, like `@field:`. The backing `field` in an accessor is an operand, and the words used as names (`val by = 1`, `x.get(i)`) stay - operands. + operands. #1565 made Java and Groovy bill every modifier, where they + had billed only `abstract`, `final`, `transient` and `synchronized`: + the visibility keywords, `static`, `native`, `strictfp`, `volatile`, + `sealed` and `non-sealed`, plus the `permits` clause keyword beside + `extends` and `implements`. Java's are the keywords, never their + `modifiers` wrapper. The `static` of `import static` and `requires + static` bills nothing, as `import` does not, and the words used as + names (`var sealed = 1`) stay operands. - **A generic list's angle brackets are one bracket pair.** The `<` and `>` around type arguments and type parameters count as one `<>` operator, billed at the `<` as `()` is at the `(`, and the `>` counts diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 49ea2721..a80551ed 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -2684,27 +2684,30 @@ mod tests { }", "foo.java", |metric| { - // Operators (n1=12): class {} void () [] , . ; int = + / + // Operators (n1=14): public static class {} void () [] , . ; int = + / + // (`public` x2 and `static` since #1565) // Operands (n2=13): Main main string args a b c avg 5 3 MessageFormat format "{0}" // (`string` is the parameter's type name, an operand since #1560) + assert_eq!(metric.halstead.unique_operators(), 14); + assert_eq!(metric.halstead.total_operators(), 30); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 12, - "total_operators": 27, + "unique_operators": 14, + "total_operators": 30, "unique_operands": 13, "total_operands": 23, - "length": 50, - "estimated_program_length": 91.12526634448807, - "purity_ratio": 1.8225053268897613, - "vocabulary": 25, - "volume": 232.19280948873623, - "difficulty": 10.615384615384615, - "level": 0.09420289855072464, - "effort": 2464.8159776496614, - "time": 136.93422098053674, - "bugs": 0.06082308053869885 + "length": 53, + "estimated_program_length": 101.40868524464065, + "purity_ratio": 1.913371419710201, + "vocabulary": 27, + "volume": 252.0090376146638, + "difficulty": 12.384615384615385, + "level": 0.08074534161490683, + "effort": 3121.035004304683, + "time": 173.3908335724824, + "bugs": 0.07118871793971378 } "# ); @@ -2731,26 +2734,27 @@ mod tests { |metric| { // Verifies all 8 Java primitive-type keywords (byte, short, int, long, // char, float, double, boolean) are counted as distinct operators, and - // that true/false are counted as operands. The other four operators - // are `class` (#1552), `{}`, `=` and `;`. + // that true/false are counted as operands. The other five operators + // are `public` (#1565), `class` (#1552), `{}`, `=` and `;`. + assert_eq!(metric.halstead.unique_operators(), 13); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 12, - "total_operators": 29, + "unique_operators": 13, + "total_operators": 30, "unique_operands": 19, "total_operands": 19, - "length": 48, - "estimated_program_length": 123.730172764082, - "purity_ratio": 2.5777119325850415, - "vocabulary": 31, - "volume": 237.80142289857, - "difficulty": 6.0, - "level": 0.16666666666666666, - "effort": 1426.80853739142, - "time": 79.26714096619, - "bugs": 0.04224635182363915 + "length": 49, + "estimated_program_length": 128.8163390912623, + "purity_ratio": 2.6289048794135166, + "vocabulary": 32, + "volume": 245.0, + "difficulty": 6.5, + "level": 0.15384615384615385, + "effort": 1592.5, + "time": 88.47222222222223, + "bugs": 0.04545674902336332 } "# ); @@ -2952,8 +2956,9 @@ mod tests { // not already counted as operands, since `String` // was already an identifier in the prior grammar's // counting). - // `class` is the ninth unique operator since #1552. - assert_eq!(metric.halstead.unique_operators(), 9); + // `class` is the ninth unique operator since #1552, and + // `static` the tenth since #1565. + assert_eq!(metric.halstead.unique_operators(), 10); assert_eq!(metric.halstead.unique_operands(), 13); // `bugs` is `effort.powf(2/3) / 3000`, and the platform libm's // `powf` differs in the last ulp: macOS rendered this value @@ -2964,20 +2969,20 @@ mod tests { { ".bugs" => insta::rounded_redaction(12) }, @r#" { - "unique_operators": 9, - "total_operators": 23, + "unique_operators": 10, + "total_operators": 24, "unique_operands": 13, "total_operands": 23, - "length": 46, - "estimated_program_length": 76.63504134881501, - "purity_ratio": 1.665979159756848, - "vocabulary": 22, - "volume": 205.13385445731566, - "difficulty": 7.961538461538462, - "level": 0.12560386473429952, - "effort": 1633.1810720255517, - "time": 90.73228177919731, - "bugs": 0.046227630923 + "length": 47, + "estimated_program_length": 81.32499728470782, + "purity_ratio": 1.730319091163996, + "vocabulary": 23, + "volume": 212.60741193467962, + "difficulty": 8.846153846153847, + "level": 0.11304347826086956, + "effort": 1880.7578748067813, + "time": 104.48654860037674, + "bugs": 0.050788728209 } "# ); @@ -6314,6 +6319,235 @@ end", assert_operator_rows::("foo.groovy", ROWS); } + /// #1565: Java bills every modifier, as C# and Kotlin do, each as + /// its keyword leaf and never as the `modifiers` wrapper. Before the + /// fix only `abstract`, `final`, `transient`, `synchronized` and + /// `default` billed. The rows from "import static" on are controls: + /// a directive's `static` and the contextual words used as names add + /// no operator. + #[cfg(feature = "java")] + #[test] + fn java_modifiers_are_operators_1565() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}`, `public`, `private`, `protected`, `int` x3, + // `;` x3 + ( + "visibility", + "class C { public int a; private int b; protected int c; }\n", + [7, 11], + ), + // `class`, `{}`, `static`, `final`, `int` x2, `=`, `;` x2, + // `transient`, `volatile` + ( + "field modifiers", + "class C { static final int a = 1; transient volatile int b; }\n", + [9, 11], + ), + // `class`, `{}` x3, `native`, `synchronized`, `strictfp`, + // `void` x3, `()` x3, `;` + ( + "method modifiers", + "class C { native void n(); synchronized void s() {} strictfp void f() {} }\n", + [8, 14], + ), + // `sealed`, `class` x2, `permits`, `{}` x2, `non-sealed`, + // `extends` + ( + "sealed hierarchy", + "sealed class S permits T {}\nnon-sealed class T extends S {}\n", + [6, 8], + ), + // `class`, `{}` x2, `static`: the initializer's `static` + ("static initializer", "class C { static {} }\n", [3, 4]), + // `class`, `{}` x2, `public`, `()`, `return`, `;`: the + // annotation beside the modifier bills nothing of its own + ( + "annotation in modifiers", + "class C { @Override public String toString() { return s; } }\n", + [6, 7], + ), + // `.` x3, `;`, `class`, `{}`: no `static` + ( + "import static", + "import static java.lang.Math.max;\nclass C {}\n", + [4, 6], + ), + // `{}`, `;`: no `static`, and `transitive` stays unbilled + // with the rest of the module directive + ( + "requires static", + "module m { requires static transitive a; }\n", + [2, 2], + ), + // `class`, `{}` x2, `void`, `()`, `=` x3, `;` x3, `int` + ( + "names", + "class C { void m() { var sealed = 1; int permits = 2; sealed = permits; } }\n", + [7, 12], + ), + // `class`, `{}`, `int` x2, `,`, `;` x2, `=`, `-` + ( + "non-sealed as a subtraction", + "class C { int non, sealed; int z = non-sealed; }\n", + [7, 9], + ), + ]; + assert_operator_rows::("Foo.java", ROWS); + } + + /// #1565: the issue's fixture, by vocabulary. Each modifier is + /// listed by its own word, never by `modifiers`, and the contextual + /// words used as names are operands only. No getter edit makes that + /// second half fail, since Java's arms carry no guard on those + /// words: it pins the grammar emitting an `identifier` for a name, + /// so a bump that routes one to the keyword kind trips it. + #[cfg(feature = "java")] + #[test] + fn java_modifier_vocabulary_1565() { + let ops = ops_of::( + "public abstract class C { private static final int a = 1; \ + protected transient int b; public synchronized void m() {} }\n", + "Foo.java", + ); + assert_eq!( + ops.operators, + [ + "()", + ";", + "=", + "abstract", + "class", + "final", + "int", + "private", + "protected", + "public", + "static", + "synchronized", + "transient", + "void", + "{}", + ] + ); + assert_eq!(ops.operands, ["1", "C", "a", "b", "m"]); + + assert_keywords_are_operands_only::( + "class C { int sealed() { return permits; } void m() { sealed(); } }\n", + "Foo.java", + &["sealed", "permits"], + ); + } + + /// #1565: Groovy bills every modifier, as Java now does. Its grammar + /// has no `modifiers` wrapper, so each leaf sits directly under its + /// declaration. `void` and `int` are type names and bill as + /// operands, so they are absent from these operator counts. The last + /// two rows are controls. + #[cfg(feature = "groovy")] + #[test] + fn groovy_modifiers_are_operators_1565() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}`, `public`, `private`, `protected`, `def` x3 + ( + "visibility", + "class C {\n public def a\n private def b\n protected def c\n}\n", + [6, 8], + ), + // `class`, `{}`, `static`, `final`, `def` x2, `=`, + // `transient`, `volatile` + ( + "field modifiers", + "class C {\n static final def a = 1\n transient volatile def b\n}\n", + [8, 9], + ), + // `class`, `{}` x3, `native`, `synchronized`, `strictfp`, + // `()` x3 + ( + "method modifiers", + "class C {\n native void n()\n synchronized void s() {}\n \ + strictfp void f() {}\n}\n", + [6, 10], + ), + // `sealed`, `class` x2, `permits`, `{}` x2, `non-sealed`, + // `extends` + ( + "sealed hierarchy", + "sealed class S permits T {}\nnon-sealed class T extends S {}\n", + [6, 8], + ), + // `sealed` x2, `interface`, `trait`, `permits` x2, `{}` x2 + ( + "sealed interface and trait", + "sealed interface I permits A {}\nsealed trait T permits A {}\n", + [5, 8], + ), + // `class`, `{}` x3, `static` x2, `()`: an initializer and a + // script-level method + ( + "static initializer and method", + "class C {\n static {}\n}\nstatic void top() {}\n", + [4, 7], + ), + // `.` x3: no `static`, as `import` bills nothing + ( + "import static", + "import static java.lang.Math.max\n", + [1, 3], + ), + // `=` x2, `.`, `()` x2, `def`: a statement opening with + // `sealed` as a name fails into an `ERROR` holding the + // keyword leaf, which bills nothing + ( + "contextual names", + "sealed = 1\nsealed.call()\ndef permits = 2\nfoo(permits)\n", + [4, 6], + ), + ]; + assert_operator_rows::("foo.groovy", ROWS); + } + + /// #1565: the issue's Groovy fixture, by vocabulary, plus the + /// modifier words in positions the grammar parses as a name. Unlike + /// Java, Groovy accepts even its hard keywords as a map key, a named + /// argument and a member name. No getter edit makes that half fail: + /// it pins the grammar emitting an `identifier` there, so a bump + /// that routes one to the keyword kind trips it. + #[cfg(feature = "groovy")] + #[test] + fn groovy_modifier_vocabulary_1565() { + let ops = ops_of::( + "public abstract class C { private static final def a = 1; protected def m() {} }\n", + "foo.groovy", + ); + assert_eq!( + ops.operators, + [ + "()", + ";", + "=", + "abstract", + "class", + "def", + "final", + "private", + "protected", + "public", + "static", + "{}", + ] + ); + assert_eq!(ops.operands, ["1", "C", "a", "m"]); + + assert_keywords_are_operands_only::( + "x = sealed\nfoo(permits)\ndef m = [static: 1, public: 2]\n\ + foo(native: 1, volatile: 2)\ny = obj.private\n", + "foo.groovy", + &[ + "sealed", "permits", "static", "public", "native", "volatile", "private", + ], + ); + } + /// #1552 rows shared by the two C++ clones. The template-parameter /// rows pin the synonym: `class T` is `typename T`, and the two /// spellings score alike. Both were unbilled until #1557 billed diff --git a/src/ops.rs b/src/ops.rs index 137982c5..512ca4e6 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -902,8 +902,10 @@ mod tests { } }", "foo.java", + // `public` and `static` are billed since #1565 &mut [ - "class", "{}", "void", "()", "[]", ",", ".", ";", "int", "=", "+", "/", + "public", "static", "class", "{}", "void", "()", "[]", ",", ".", ";", "int", "=", + "+", "/", ], &mut [ "Main", @@ -942,8 +944,10 @@ mod tests { }", "foo.java", // All 8 primitive-type keywords must appear as distinct operators. - // true/false appear as operands; `class` is billed since #1552. + // true/false appear as operands; `class` is billed since #1552 + // and `public` since #1565. &mut [ + "public", "class", "{}", ";", From 4b2e686ea8e596e57fd46f1ea45c3822710e83db Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:24:01 -0700 Subject: [PATCH 04/34] fix(getter/java): bill a class literal's parts A Java class literal billed its whole text as one operand while the `.`, `[]` and primitive inside it also billed as operators, and the `class` tail and type names were suppressed: `int.class` paid `int` and `.` on top of the `int.class` operand that already spelled them. JLS 15.8.2 defines the class literal as a primary expression, not a literal (3.10 lists none), so its tokens are its vocabulary. The `class_literal` kind leaves the operand arm, the #1560 type-name guard is deleted so the type bills as everywhere else, and the `class` tail becomes an operand, as Groovy's `Foo.class` and Kotlin's and PHP's `Foo::class` bill theirs. The declaration keyword stays an operator. `java.util.List.class` now bills three `.` and operands `java`, `util`, `List`, `class`; `Foo[].class` bills `[]`, `.`, `Foo`, `class`; `int.class` / `void.class` bill `int` / `void`, `.` and `class`. The shared shapes are asserted against Groovy from one table. No Java corpus file is snapshotted, so no integration snapshot moves. Fixes #1566 --- big-code-analysis-ast/src/getter/java.rs | 33 ++--- big-code-analysis-book/src/metrics.md | 16 ++- src/metrics/halstead.rs | 149 +++++++++++++++++++++-- 3 files changed, 159 insertions(+), 39 deletions(-) diff --git a/big-code-analysis-ast/src/getter/java.rs b/big-code-analysis-ast/src/getter/java.rs index 073ba2d5..62d87fbb 100644 --- a/big-code-analysis-ast/src/getter/java.rs +++ b/big-code-analysis-ast/src/getter/java.rs @@ -96,26 +96,14 @@ impl Getter for JavaCode { // https://www.geeksforgeeks.org/software-engineering-halsteads-software-metrics/?msclkid=5e181114abef11ecbb03527e95a34828 match node.kind_id().into() { // `class` declares a class (billed below, #1552) except as the - // tail of a class literal (`C.class`): that `class_literal` is - // already one operand keyed by its whole text, so billing its - // leaf too would pay the same bytes twice (grammar-dispatch - // section 5, the compound-leaf guard). - Class if ancestors.parent_has_kind(node, ClassLiteral as u16) => TokenRole::Unknown, - // A type name is an operand (#1560, the operand arm below) - // except inside a class literal, whose whole text is already - // one operand — the `class` guard above, for the same - // reason. The type sits there directly (`Foo.class`) or - // under an array or scoped type (`Foo[].class`, - // `java.util.List.class`), so the climb skips those two. - TypeIdentifier - if ancestors - .iter(node) - .map(|(ancestor, _)| ancestor.kind_id()) - .find(|&id| id != ArrayType as u16 && id != ScopedTypeIdentifier as u16) - == Some(ClassLiteral as u16) => - { - TokenRole::Unknown - } + // tail of a class literal (`C.class`), where it is an operand: + // JLS §15.8.2 makes the literal a primary expression, not a + // literal (§3.10), so its parts bill as they would anywhere + // else — the type name an operand, the primitive or `void` + // and the `.` / `[]` operators — and the `class` tail is the + // operand they act on, as Kotlin's and PHP's `Foo::class` and + // Groovy's `Foo.class` bill theirs (#1566). + Class if ancestors.parent_has_kind(node, ClassLiteral as u16) => TokenRole::Operand, // `static` is a modifier (below, #1565) except in an import // or module directive (`import static`, `requires static`), // whose own keyword — `import`, `requires` — bills nothing @@ -189,10 +177,11 @@ impl Getter for JavaCode { // Go, Groovy and TypeScript bill type names: a field or // parameter type, an `implements` / `extends` entry, a type // parameter, a generic argument, each segment of - // `java.util.List` (the `.` bills as itself). It is the one + // `java.util.List` (the `.` bills as itself), and since #1566 + // the type of a class literal (`Foo.class`). It is the one // type-name kind and no listed node wraps it, so each name // bills once. - Identifier | TypeIdentifier | NullLiteral | ClassLiteral | True | False + Identifier | TypeIdentifier | NullLiteral | True | False | StringLiteral | CharacterLiteral | HexIntegerLiteral | OctalIntegerLiteral | BinaryIntegerLiteral | DecimalIntegerLiteral | HexFloatingPointLiteral | DecimalFloatingPointLiteral | This => { diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index fc04b2b8..6317f513 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -715,7 +715,8 @@ tokens involved the same way they spell real operators: the JavaScript family, C, C++ and Objective-C followed in #1552 with their type-declaration keywords: `class`, `struct`, `union`, `enum`, `interface`, `record`, `trait`, `@interface`, `@implementation` and - `@protocol`. Java's `C.class` is not billed: it stays one operand. + `@protocol`. The `class` tail of Java's `C.class` is an operand, not + this keyword (#1566, below). Kotlin's `interface` followed in #1556, and since #1558 Kotlin bills every modifier, as C# does: the `enum` of `enum class`, `data`, `sealed`, the visibility keywords, `open`, @@ -732,10 +733,15 @@ tokens involved the same way they spell real operators: TypeScript type name an operand (`x: Foo`, `interface I`, `Map`), as C#, Rust and the C family already did. Java followed in #1560: `Foo f`, `implements I`, `List` and each segment of - `java.util.List` bill their names as operands. A type inside a class - literal (`Foo.class`) does not, since the literal is already one - operand, and neither does the inferred type `var`, which C# bills - nothing for either. #1561 made TypeScript's type-level operator + `java.util.List` bill their names as operands. The inferred type + `var` does not, as C# bills nothing for its own. Since #1566 a Java + class literal bills its parts rather than one operand spelled by its + whole text, since JLS §15.8.2 defines it as an expression rather than + a literal: in `java.util.List.class` and `Foo[].class` each name and + the `class` tail are operands and each `.` and `[]` an operator, as + Groovy's `Foo.class` and Kotlin's and PHP's `Foo::class` bill theirs. + The `int` of `int.class` and the `void` of `void.class` stay + operators, as Java bills them everywhere. #1561 made TypeScript's type-level operator keywords operators, as their value-level siblings `as`, `typeof` and `extends` already were: `satisfies`, `keyof`, `infer`, and the `is` and `asserts` of a type predicate. #1563 made TypeScript's diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index a80551ed..2ce3af7c 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -6203,8 +6203,8 @@ end", } /// #1552: Java's type-declaration keywords are operators, as C#'s - /// are. The class-literal row pins the guard: `C.class` is one - /// operand, and its `class` leaf bills nothing of its own. + /// are. The class-literal row pins the guard: the `class` tail of + /// `C.class` is an operand (#1566), not a declaration keyword. #[cfg(feature = "java")] #[test] fn java_declaration_keywords_are_operators_1552() { @@ -6220,7 +6220,7 @@ end", // `@interface`, `{}` ("annotation type", "@interface N {}\n", [2, 2]), // `class`, `{}`, `=`, `.`, `;` — one `class`: the literal's - // is part of the `C.class` operand + // tail is an operand (#1566) ("class literal", "class U { Object o = C.class; }\n", [5, 5]), ]; assert_operator_rows::("Foo.java", ROWS); @@ -6229,9 +6229,9 @@ end", /// #1560: a Java type name is an operand, as in C#, Rust, the C /// family, Go, Groovy and TypeScript. Each row isolates one position /// of the single `type_identifier` kind. The class-literal rows pin - /// that a type inside `Foo.class` is not billed again on top of the - /// literal's own operand, whether it sits there directly or under an - /// array or scoped type; the `var` rows, that the inferred-type + /// that a type inside `Foo.class` bills like any other (#1566), + /// whether it sits there directly or under an array or scoped type, + /// beside the literal's `class` tail; the `var` rows, that the inferred-type /// keyword bills nothing while a variable named `var` stays an /// operand. #[cfg(feature = "java")] @@ -6258,23 +6258,23 @@ end", ), // `C`, `m`, `Ex` ("throws", "class C { void m() throws Ex {} }\n", [3, 3]), - // `C`, `Object`, `a`, `Foo.class` + // `C`, `Object`, `a`, `Foo`, `class` ( "class literal", "class C { Object a = Foo.class; }\n", - [4, 4], + [5, 5], ), - // `C`, `Object`, `a`, `Foo[].class` + // `C`, `Object`, `a`, `Foo`, `class` ( "array class literal", "class C { Object a = Foo[].class; }\n", - [4, 4], + [5, 5], ), - // `C`, `Object`, `a`, `java.util.List.class` + // `C`, `Object`, `a`, `java`, `util`, `List`, `class` ( "scoped class literal", "class C { Object a = java.util.List.class; }\n", - [4, 4], + [7, 7], ), // `C`, `m`, `x`, `1` — no `var` ( @@ -6292,6 +6292,131 @@ end", assert_operand_rows::("Foo.java", ROWS); } + /// One class literal's own vocabulary: the source spelling, then the + /// operators and operands it adds to the fixture's frame. + #[cfg(any(feature = "groovy", feature = "java"))] + type ClassLiteralParts = ( + &'static str, + &'static [&'static str], + &'static [&'static str], + ); + + /// #1566: the class-literal shapes Java and Groovy spell alike. The + /// expected parts are what Groovy — untouched by #1566 — already + /// billed for `Foo.class`: the type name an operand, the `.` and + /// `[]` operators, and the `class` tail an operand. Both tests below + /// read this one table, so the two languages cannot drift apart. + #[cfg(any(feature = "groovy", feature = "java"))] + const SHARED_CLASS_LITERALS: &[ClassLiteralParts] = &[ + ("Foo.class", &["."], &["Foo", "class"]), + ("Foo[].class", &[".", "[]"], &["Foo", "class"]), + ( + "java.util.List.class", + &["."], + &["List", "class", "java", "util"], + ), + ]; + + /// Asserts each literal's operator and operand vocabularies inside + /// `frame`, which spells the fixture around it and contributes + /// `frame_ops` / `frame_operands` of its own. + #[cfg(any(feature = "groovy", feature = "java"))] + #[track_caller] + fn assert_class_literal_parts( + file: &str, + frame: fn(&str) -> String, + frame_ops: &[&str], + frame_operands: &[&str], + rows: &[ClassLiteralParts], + ) { + let merged = |frame: &[&str], parts: &[&str]| { + let mut all: Vec = frame.iter().chain(parts).map(|s| (*s).to_owned()).collect(); + all.sort_unstable(); + all.dedup(); + all + }; + for (literal, ops, operands) in rows { + let got = ops_of::(&frame(literal), file); + assert_eq!( + got.operators, + merged(frame_ops, ops), + "operators of {literal}" + ); + assert_eq!( + got.operands, + merged(frame_operands, operands), + "operands of {literal}" + ); + } + } + + /// #1566: a Java class literal bills its parts. JLS §15.8.2 makes it + /// a primary expression rather than a literal (§3.10 lists none), so + /// the type name is an operand as anywhere else, `.` and `[]` are + /// operators, and the `class` tail is the operand they act on — the + /// split Kotlin's and PHP's `Foo::class` and Groovy's `Foo.class` + /// already made. Before the fix the literal billed as one operand + /// spelled by its whole text while `.`, `[]` and a primitive inside + /// it also billed as operators. The shared rows take their expected + /// values from Groovy; the primitive and `void` rows are Java's own, + /// since Groovy still bills `int` and `void` as operands (#1568). + #[cfg(feature = "java")] + #[test] + fn java_class_literal_bills_its_parts_1566() { + const FRAME_OPS: &[&str] = &[";", "=", "class", "{}"]; + const FRAME_OPERANDS: &[&str] = &["C", "Object", "c"]; + let frame = |literal: &str| format!("class C {{ Object c = {literal}; }}\n"); + assert_class_literal_parts::( + "Foo.java", + frame, + FRAME_OPS, + FRAME_OPERANDS, + SHARED_CLASS_LITERALS, + ); + assert_class_literal_parts::( + "Foo.java", + frame, + FRAME_OPS, + FRAME_OPERANDS, + &[ + ("int.class", &[".", "int"], &["class"]), + ("void.class", &[".", "void"], &["class"]), + ], + ); + // The totals pin the multiplicities a vocabulary cannot: the frame + // bills four operators (`class`, `{}`, `=`, `;`) and three operands + // (`C`, `Object`, `c`), and each literal adds its parts once — + // `java.util.List.class` its three `.` and four names. + assert_halstead_rows::( + "Foo.java", + &[ + ("int", "class C { Object c = int.class; }\n", [6, 4]), + ("array", "class C { Object c = Foo[].class; }\n", [6, 5]), + ( + "scoped", + "class C { Object c = java.util.List.class; }\n", + [7, 7], + ), + ("void", "class C { Object c = void.class; }\n", [6, 4]), + ], + |h| [h.total_operators(), h.total_operands()], + ); + } + + /// #1566's oracle: Groovy bills the shared class-literal shapes the + /// way Java now does. A Groovy field needs no `;`. + #[cfg(feature = "groovy")] + #[test] + fn groovy_class_literal_bills_its_parts_1566() { + assert_class_literal_parts::( + "foo.groovy", + |literal| format!("class C {{ Object c = {literal} }}\n"), + &["=", "class", "{}"], + &["C", "Object", "c"], + SHARED_CLASS_LITERALS, + ); + } + /// #1552: Groovy's type-declaration keywords are operators, as /// Java's are. `String.class` is a `field_access` whose `class` is a /// plain identifier, so it stays an operand with no guard needed. From 41a6b0bd4032b8d567a40cec836c5a661d8f6ac7 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:37:37 -0700 Subject: [PATCH 05/34] fix(halstead): bill type-enclosing angles as <> #1559 billed a generic list's angle brackets as one `<>` pair. Three bracket pairs that enclose types but are not list nodes still billed the comparison `<` and `>`: a Rust qualified path's bracketed_type (`::f`, `::f`), a C# function_pointer_type (`delegate*`) and a Kotlin super_expression (`super`). Each parent holds `<` / `>` only as its own brackets, so adding it to the grammar's GENERIC_ANGLES.lists cannot capture a comparison. The rule is now "angle brackets that enclose types are one bracket pair". Kotlin's row leaves the ABC/Halstead bracket parity test, which excludes generic brackets by design; ABC's half stays pinned by kotlin_super_type_argument_is_not_a_condition. Nine serde corpus snapshots move in halstead.* and mi.* only. Fixes #1567 --- big-code-analysis-ast/src/getter.rs | 5 +-- big-code-analysis-ast/src/getter/csharp.rs | 3 ++ .../src/getter/generic_angles.rs | 14 ++++--- big-code-analysis-ast/src/getter/kotlin.rs | 10 +++-- big-code-analysis-ast/src/getter/rust.rs | 4 +- big-code-analysis-book/src/metrics.md | 21 +++++----- src/ops.rs | 40 +++++++++++++++++++ tests/parity/abc_halstead_bracket_parity.rs | 30 ++++---------- tests/parity/main.rs | 1 - tests/repositories/big-code-analysis-output | 2 +- 10 files changed, 83 insertions(+), 47 deletions(-) diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index 85278aae..e935c063 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -232,8 +232,7 @@ pub trait Getter { /// counts the alphabet a program is written in and has no notion of /// a decision, so a `<` is the same vocabulary entry whether it /// opens a JSX tag, brackets a Lua `` attribute, names a C# - /// `operator <` overload, disambiguates a Kotlin `super`, or - /// delimits a Perl `` readline. Every impl that classifies + /// `operator <` overload, or delimits a Perl `` readline. Every impl that classifies /// punctuation at all bills `(`, `{`, `[`, `,`, `;`, `.` and `:` on /// exactly that reading, for each glyph its language spells. Python /// and Rust were the last two exceptions, closed by #1486. @@ -241,7 +240,7 @@ pub trait Getter { /// A *decision* metric gates the same token by role, and is /// supposed to disagree: ABC's `conditions` counts a `<` only under /// a `binary_expression` (#1274, #1275, #1280, #1297), so each of - /// the five constructs above scores zero there while `bca ops` + /// the four constructs above scores zero there while `bca ops` /// still reports the bracket. #1395 settled that the two answers /// are both correct rather than a drift to be reconciled; /// `tests/parity/abc_halstead_bracket_parity.rs` fails if a later diff --git a/big-code-analysis-ast/src/getter/csharp.rs b/big-code-analysis-ast/src/getter/csharp.rs index 00024916..4d5f94f3 100644 --- a/big-code-analysis-ast/src/getter/csharp.rs +++ b/big-code-analysis-ast/src/getter/csharp.rs @@ -3,10 +3,13 @@ use super::*; +// A function pointer type's `<>` (`delegate*`) encloses +// its parameter and return types (#1567). const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { lists: &[ Csharp::TypeArgumentList as u16, Csharp::TypeParameterList as u16, + Csharp::FunctionPointerType as u16, ], openers: &[Csharp::LT as u16], closers: &[Csharp::GT as u16], diff --git a/big-code-analysis-ast/src/getter/generic_angles.rs b/big-code-analysis-ast/src/getter/generic_angles.rs index ff67d09a..60e6afb2 100644 --- a/big-code-analysis-ast/src/getter/generic_angles.rs +++ b/big-code-analysis-ast/src/getter/generic_angles.rs @@ -10,18 +10,22 @@ use crate::{Ancestors, Node}; pub(crate) const ANGLE_PAIR: &str = "<>"; /// The kinds one grammar spells its generic angle brackets with: the -/// list nodes whose `<` / `>` delimit type arguments or parameters -/// rather than compare, and the bracket tokens themselves. +/// nodes whose `<` / `>` enclose types rather than compare, and the +/// bracket tokens themselves. /// -/// A list's brackets are one bracket pair, billed like `()`, `[]` and +/// Angle brackets that enclose types are one bracket pair, billed like `()`, `[]` and /// `{}` (#1395): the opener carries the pair as [`ANGLE_PAIR`] and the /// closer bills nothing (#1559). The grammars give the brackets the /// same kind ids as the comparison operators, so only the parent tells /// the two apart, and the opener is keyed by text rather than by kind /// to keep it out of the comparison `<`'s vocabulary entry. pub(crate) struct GenericAngleKinds { - /// The list nodes: type-argument, type-parameter and template - /// lists, plus each grammar's other spelling of a generic list. + /// The nodes whose brackets enclose types: type-argument, + /// type-parameter and template lists, each grammar's other spelling + /// of a generic list, and the single-type brackets of a Rust + /// qualified path (`::f`), a C# function pointer + /// (`delegate*`) and a Kotlin `super` (#1567). Each + /// has `<` / `>` only as its own brackets. pub(crate) lists: &'static [u16], /// The `<` kinds, with any alias the grammar declares for it. pub(crate) openers: &'static [u16], diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index 83571b56..d177523a 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -175,10 +175,14 @@ fn kotlin_next_token_is_inserted<'a>(node: &Node<'a>, ancestors: Ancestors<'a, ' }) } -// `super` (a `super_expression`) names a supertype rather than -// listing type arguments, so its brackets stay `<` and `>`. +// A qualified `super` (a `super_expression`) encloses the supertype +// it names (#1567). const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { - lists: &[Kotlin::TypeArguments as u16, Kotlin::TypeParameters as u16], + lists: &[ + Kotlin::TypeArguments as u16, + Kotlin::TypeParameters as u16, + Kotlin::SuperExpression as u16, + ], openers: &[Kotlin::LT as u16], closers: &[Kotlin::GT as u16], is_misparse: never_misparsed, diff --git a/big-code-analysis-ast/src/getter/rust.rs b/big-code-analysis-ast/src/getter/rust.rs index be49e11f..27f5ea6e 100644 --- a/big-code-analysis-ast/src/getter/rust.rs +++ b/big-code-analysis-ast/src/getter/rust.rs @@ -5,7 +5,8 @@ use super::*; // `for_lifetimes` (`for<'a> fn(&'a u8)`) and `use_bounds` // (`impl Tr + use<'a, T>`) are generic-parameter lists under other -// names. `LT2` is the `token(prec(1, '<'))` spelling `type_arguments` +// names, and a qualified path's `bracketed_type` (`::f`, +// `::f`) encloses a type (#1567). `LT2` is the `token(prec(1, '<'))` spelling `type_arguments` // and `use_bounds` open with; the parser reports it as `LT`, so it is // listed defensively, here and in the operator arm, and its absence is // pinned by `rust_generic_opener_alias_never_reaches_kind_id`. @@ -15,6 +16,7 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { Rust::TypeParameters as u16, Rust::ForLifetimes as u16, Rust::UseBounds as u16, + Rust::BracketedType as u16, ], openers: &[Rust::LT as u16, Rust::LT2 as u16], closers: &[Rust::GT as u16], diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 6317f513..0660e6d8 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -159,7 +159,7 @@ application would over-count. | Ruby | A subject-less `case` scores each `when` pattern as an `\|\|` operand; a subject-ful `case` scores per clause | A subject-less clause (`case; when x > 5`) lists ordinary boolean expressions, evaluated like an `if` predicate, and several patterns (`when a, b`) are an implicit `\|\|`. Each pattern therefore scores once unless its own operator already counted it, so `when x > 5` scores 1 like `if x > 5`, and `when a, b` scores 2 like `if a \|\| b`. Before this, a blanket per-clause increment counted a comparison twice. A subject-ful clause (`case x; when 1`) matches each pattern with an implicit `pattern === x` that nothing in the source spells, so the clause pays once (#1453). | | Go | A tagless `switch` scores each case expression as an `\|\|` operand; a tagged `switch` scores per case | A tagless switch (`switch { case x > 5: }`, with or without an init statement) is `switch true`: each case expression is an ordinary boolean scored by the same slot an `if` predicate uses, and a list (`case a, b:`) is an implicit `\|\|`. So `case x > 5:` scores 1 like `if x > 5`, and `case a, b:` scores 2 like `if a \|\| b`. Before this, a blanket per-case increment counted a comparison twice. A tagged case (`switch x { case 1: }`) compares with an implicit `x == 1` that nothing in the source spells, so the case pays once; type-switch and `select` cases are unchanged (#1523). | | Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive` (its `after` timeout included), of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). | -| C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these brackets as an operator (a generic list's as one `<>` pair since #1559), and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | +| C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these brackets as an operator (a type-enclosing pair as one `<>` since #1559 and #1567), and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | | C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | | Perl | A comparison's right operand is not a condition, although the grammar parses it as one | tree-sitter-perl 1.1.2 lets a chaining comparison (`<`, `>`, `<=`, `>=`, `lt`, `gt`, `le`, `ge`, `==`, `!=`, `eq`, `ne`) take everything of lower precedence to its right as its operand, so `$a > $b ? 1 : 2` parses as `$a > ($b ? 1 : 2)` and `$a > $b && $c` as `$a > ($b && $c)`. The ternary's condition slot and the chain's first operand then held `$b`, which scored as a condition of its own: one above the JavaScript twin for each misparsed comparison. The walker recognises the shape and leaves that operand to the comparison, so both spellings score what the parenthesised `($a > $b) ? 1 : 2` and `($a > $b) && $c` do. An explicit `$a > ($b ? 1 : 2)` is unaffected. `<=>`, `cmp`, `=~` and `!~` do not chain and parse correctly. The workaround goes when the grammar is fixed upstream (#1273). | @@ -699,9 +699,8 @@ tokens involved the same way they spell real operators: there. Halstead counts the vocabulary a program is written in and has no notion of a decision, so a `<` is the same vocabulary entry whether it is a comparison, a JSX tag delimiter, a Lua `` - attribute, the declared name of a C# `operator <`, a Kotlin - `super`, or a Perl `` readline — and `bca ops` reports every - one of them. This is the deliberate counterpart to + attribute, the declared name of a C# `operator <`, or a Perl `` + readline — and `bca ops` reports every one of them. This is the deliberate counterpart to [the ABC deviation](#abc-per-language-deviations) that excludes the same tokens from `conditions`: ABC asks whether a branch is taken, and these are not branches; Halstead asks what alphabet the source uses, @@ -763,14 +762,16 @@ tokens involved the same way they spell real operators: `modifiers` wrapper. The `static` of `import static` and `requires static` bills nothing, as `import` does not, and the words used as names (`var sealed = 1`) stay operands. -- **A generic list's angle brackets are one bracket pair.** The `<` +- **Angle brackets that enclose types are one bracket pair.** The `<` and `>` around type arguments and type parameters count as one `<>` operator, billed at the `<` as `()` is at the `(`, and the `>` counts nothing (#1559). That covers `Vec`, `fn f`, a turbofish - `h::()`, `for<'a>` and `use` in Rust; C++'s + `h::()`, `for<'a>`, `use` and a qualified path's + `::` or `::` in Rust; C++'s `template ` and `std::vector`; and the type-argument and type-parameter lists of Java, Groovy, C#, Kotlin, TypeScript and - TSX. Objective-C has four generic lists: `NSArray`, + TSX, plus C#'s `delegate*` pointer type and Kotlin's + `super` (#1567). Objective-C has four generic lists: `NSArray`, `id`, the protocol list in `@interface A : NSObject `, and a type passed as a call argument, `f(NSArray)`. A comparison's `<` and `>` and a shift's `<<` and `>>` stay @@ -784,10 +785,8 @@ tokens involved the same way they spell real operators: The exception is C#'s `>=` spelling, `F(a < b, c >= d)`, which the grammar parses as an assignment and which still bills `<>` and `=`. Go's generics use square brackets and already billed `[]`. Angle brackets around anything - other than a generic list still bill `<` and `>`: a Rust - `::` path, a C# `delegate*` pointer type and - a Kotlin `super`, as well as the JSX, Lua and Perl constructs - above. So does a Kotlin generic call with arguments, `id(a)`, + other than types still bill `<` and `>`: the JSX, Lua and Perl + constructs above. So does a Kotlin generic call with arguments, `id(a)`, which tree-sitter-kotlin-ng parses as two comparisons (#1394), and a generic inside a Rust macro call, `vec![Vec::::new()]`, whose token tree tree-sitter-rust leaves unparsed; a `>>` closing two lists diff --git a/src/ops.rs b/src/ops.rs index 512ca4e6..8d66055e 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -1552,6 +1552,19 @@ mod tests { &["fn", "<>", "()", ":", "->", "impl", "+", "use", "{}"], [9, 10], ); + // A qualified path's `bracketed_type` encloses a type, `<>` with + // or without `as` (#1567). `<>` three times (two qualified paths + // and `Vec`), `()` three times, `::` and `;` twice; `as` still + // bills and the comparison `>` stays. Unfixed: [12, 19]. + check_generic_pair( + LANG::Rust, + "foo.rs", + "fn q() -> bool { as Default>::default(); ::g(); a > b }\n", + &[ + "fn", "()", "->", "bool", "{}", "<>", "u8", "as", "::", ";", ">", + ], + [11, 17], + ); } // expected: `<>` three times, `;` and `{}` twice each. @@ -1591,6 +1604,21 @@ mod tests { ], [10, 14], ); + // A function pointer type's brackets enclose its parameter and + // return types, `<>` with or without a calling convention (#1567). + // `;` three times; `{}`, `delegate`, `*`, `<>`, `int`, `,` and + // `void` twice. Unfixed: [16, 28], `<` and `>` three times each. + check_generic_pair( + LANG::Csharp, + "foo.cs", + "unsafe class C { delegate* p; delegate* unmanaged[Cdecl] q; \ + bool M() { return a < b && c > d; } }\n", + &[ + "unsafe", "class", "{}", "delegate", "*", "<>", "int", ",", "void", ";", "[]", + "bool", "()", "return", "<", "&&", ">", + ], + [17, 26], + ); } // expected: `<>` three times, `:` twice. TSX shares the getter macro @@ -1651,6 +1679,18 @@ mod tests { &["fun", "<>", "()", ":", "{}", "return", "<", "&&", ">"], [9, 12], ); + // A qualified `super` encloses the supertype it names (#1567). + // `()` three times (`A()`, `f()`, `.f()`), `:` and `{}` twice. + // Unfixed: [10, 16], `<` and `>` twice each. + check_generic_pair( + LANG::Kotlin, + "foo.kt", + "class C : A() { fun f(): Boolean { super.f(); return x < y && x > y } }\n", + &[ + "class", ":", "()", "{}", "fun", "<>", ".", "return", "<", "&&", ">", + ], + [11, 15], + ); } // expected: `<>` four times — Groovy spells a method's parameter diff --git a/tests/parity/abc_halstead_bracket_parity.rs b/tests/parity/abc_halstead_bracket_parity.rs index 8aa80317..e556fc64 100644 --- a/tests/parity/abc_halstead_bracket_parity.rs +++ b/tests/parity/abc_halstead_bracket_parity.rs @@ -49,12 +49,14 @@ use super::support::{metrics_space, ops_space}; /// for each rather than a `None` arm asserting they have no such /// construct, which would be false. What this pins is the policy, and a /// policy flip lands in a shared getter macro or a shared ABC arm, -/// which these seven rows already reach. Generic and template brackets -/// are the one family left out on purpose: Halstead bills them as a -/// single `<>` pair since #1559, which the `*_generic_angles_*` tests in -/// `src/ops.rs` pin per language. +/// which these six rows already reach. Angle brackets that enclose +/// types are the one family left out on purpose: Halstead bills them as +/// a single `<>` pair since #1559, which the `*_generic_angles_*` tests +/// in `src/ops.rs` pin per language. Kotlin's `super` joined that +/// family in #1567; its ABC half is +/// `kotlin_super_type_argument_is_not_a_condition`. // -// Two gates, one condition. The `mod` gate in `main.rs` names the seven +// Two gates, one condition. The `mod` gate in `main.rs` names the six // languages with a `Some` row and is the one that decides whether any of // this compiles: edit it when a row is added or removed. This gate and // the test's are the whole-roster union `check-test-lang-gates` derives @@ -147,22 +149,6 @@ fn fixture(lang: LANG) -> Option<(&'static str, &'static str, &'static [&'static .as_slice(), 22, ), - // A qualified super call disambiguates its supertype with the - // same two bare tokens. `override` bills as Kotlin's other - // modifiers do since #1558. - // - // expected N1 = 14: the header's `class`, `:`, `()` and `{}`, - // then `override`, `fun`, the parameter `()`, the return-type - // `:`, the body `{}`, `return`, `<`, `>`, `.` and the call's `()`. - LANG::Kotlin => ( - "class B : A() {\n override fun g(): Int { return super.g() }\n}\n", - "kt", - [ - "()", ".", ":", "<", ">", "class", "fun", "override", "return", "{}", - ] - .as_slice(), - 14, - ), // A filehandle readline. `` is deliberately not the // fixture: the grammar lexes it as one `standard_input` token, // so it has no brackets to classify and the row would assert @@ -262,7 +248,7 @@ fn a_non_comparison_bracket_is_a_halstead_operator_and_not_an_abc_condition() { } // Every language is feature-gated, so a build enabling none of the - // seven leaves a zero-iteration loop and a test that reports green + // six leaves a zero-iteration loop and a test that reports green // while asserting nothing. What keeps this from firing spuriously // is the `#[cfg(any(…))]` on this file's `mod` declaration in // `tests/parity/main.rs`, which names exactly the features whose diff --git a/tests/parity/main.rs b/tests/parity/main.rs index 7d2f25d6..20e1b0f1 100644 --- a/tests/parity/main.rs +++ b/tests/parity/main.rs @@ -27,7 +27,6 @@ mod support; #[cfg(any( feature = "csharp", feature = "javascript", - feature = "kotlin", feature = "lua", feature = "mozjs", feature = "perl", diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index 0708c3d9..2128d6da 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit 0708c3d941e278d1c304fd981cfc3f8b7752fe22 +Subproject commit 2128d6dad3f59d5ad973696123ecd99b296e0797 From 5b5d540bb75c0d37681e047ff164d725cc91acbb Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:53:58 -0700 Subject: [PATCH 06/34] fix(getter/groovy): bill primitive types as operators Groovy billed `int`, `boolean`, `void` and the other primitive type names as operands while Java bills them as operators, so identical declarations scored differently by language. The dekobon grammar has no primitive node: a declared type is a `type_identifier`, and a class literal's receiver (`int.class`) or an uninitialised local (`int t`, parsed as two expression statements) is a plain `identifier`. `get_op_type_with_code` now bills either kind as an operator when its text is one of Java's eight primitive keywords or `void`, and `is_primitive` keys it by that text. Groovy reserves these words, so a name can spell one only as a member selector after a `.`-family operator or as a map or named-argument label; those stay operands. Java's `boolean` is now listed as `boolean` rather than its node kind `boolean_type`, so `bca ops` reports both languages alike. The shared Java/Groovy class-literal table takes the `int.class`, `int[].class` and `void.class` rows. No corpus snapshot moves: no Java or Groovy corpus is snapshotted. Fixes #1568 --- big-code-analysis-ast/src/checker/groovy.rs | 13 ++ big-code-analysis-ast/src/getter/groovy.rs | 55 +++++ big-code-analysis-ast/src/getter/java.rs | 3 + big-code-analysis-book/src/metrics.md | 9 +- src/metrics/halstead.rs | 226 ++++++++++++++------ src/ops.rs | 32 +-- 6 files changed, 241 insertions(+), 97 deletions(-) diff --git a/big-code-analysis-ast/src/checker/groovy.rs b/big-code-analysis-ast/src/checker/groovy.rs index 62cc67f1..72e43ed5 100644 --- a/big-code-analysis-ast/src/checker/groovy.rs +++ b/big-code-analysis-ast/src/checker/groovy.rs @@ -43,6 +43,19 @@ impl Checker for GroovyCode { ) } + // A primitive type is a name kind spelled `int`, `void`, … (#1568), + // which only the bytes can tell, and this predicate gets none. It is + // consulted only for a node `GroovyCode::get_op_type_with_code` + // already billed as an operator, and that bills these two kinds as + // one only for a primitive spelling, so the kind alone suffices to + // key each primitive by its text. + fn is_primitive(node: &Node) -> bool { + matches!( + node.kind_id().into(), + Groovy::Identifier | Groovy::TypeIdentifier + ) + } + fn is_closure<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> bool { matches!(node.kind_id().into(), Groovy::Closure) } diff --git a/big-code-analysis-ast/src/getter/groovy.rs b/big-code-analysis-ast/src/getter/groovy.rs index 115e739e..cb289099 100644 --- a/big-code-analysis-ast/src/getter/groovy.rs +++ b/big-code-analysis-ast/src/getter/groovy.rs @@ -48,6 +48,47 @@ fn contextual_keyword_owns<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> }) } +/// Java's eight primitive type keywords and `void`, which Groovy +/// reserves as Java does. +const PRIMITIVE_TYPES: &[&str] = &[ + "boolean", "byte", "char", "double", "float", "int", "long", "short", "void", +]; + +/// Whether `node` spells a primitive type, which bills as a text-keyed +/// operator as Java's primitive keywords do (#1568). +/// +/// The pinned grammar has no primitive node: a declared type is a +/// `type_identifier` (`int x = 1`, `void m()`), and a class literal's +/// receiver (`int.class`) or an uninitialised local (`int t`, which +/// parses as two expression statements) is a plain `identifier`. Groovy +/// reserves these keywords, so a name can spell one only as a member +/// selector after a `.`-family operator (`x.int`, `x?.int`, `x::int`) or +/// as a map or named-argument label (`[int: 1]`, `f(int: 1)`); those +/// stay operands. +fn spells_primitive_type<'a>(node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>) -> bool { + use Groovy::*; + + if !matches!(node.kind_id().into(), Identifier | TypeIdentifier) + || !node_text(code, node).is_some_and(|text| PRIMITIVE_TYPES.contains(&text)) + { + return false; + } + ancestors.parent(node).is_none_or(|parent| { + let is_first_child = parent.child(0).is_some_and(|first| first.id() == node.id()); + match parent.kind_id().into() { + FieldAccess + | SafeNavigationExpression + | SafeChainDotExpression + | SpreadDotExpression + | DirectFieldAccessExpression + | MethodPointerExpression + | MethodReferenceExpression => is_first_child, + MapEntry | NamedArgument => !is_first_child, + _ => true, + } + }) +} + impl Getter for GroovyCode { /// Names the space, synthesising one for constructs that carry no /// name token (#1184). @@ -303,6 +344,20 @@ impl Getter for GroovyCode { } } + // Only the bytes tell `int` from `Integer`, so the byte-less + // `get_op_type` above still answers `Operand` for a primitive; the + // walk calls this spelling (grammar-dispatch section 7). + fn get_op_type_with_code<'a>( + node: &Node<'a>, + code: &[u8], + ancestors: Ancestors<'a, '_>, + ) -> TokenRole { + if spells_primitive_type(node, code, ancestors) { + return TokenRole::Operator; + } + Self::get_op_type(node, ancestors) + } + fn get_operator_spelling<'a>( node: &Node<'a>, ancestors: Ancestors<'a, '_>, diff --git a/big-code-analysis-ast/src/getter/java.rs b/big-code-analysis-ast/src/getter/java.rs index 62d87fbb..e2b93b2d 100644 --- a/big-code-analysis-ast/src/getter/java.rs +++ b/big-code-analysis-ast/src/getter/java.rs @@ -226,6 +226,9 @@ impl Getter for JavaCode { Java::LBRACK => "[]", Java::LBRACE => "{}", Java::VoidType => "void", + // The kind is `boolean_type`; spelled as written, as Groovy's + // text-keyed primitives are (#1568). + Java::BooleanType => "boolean", _ => typ.into(), } } diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 0660e6d8..22e2a691 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -740,7 +740,14 @@ tokens involved the same way they spell real operators: the `class` tail are operands and each `.` and `[]` an operator, as Groovy's `Foo.class` and Kotlin's and PHP's `Foo::class` bill theirs. The `int` of `int.class` and the `void` of `void.class` stay - operators, as Java bills them everywhere. #1561 made TypeScript's type-level operator + operators, as Java bills them everywhere. Groovy, whose grammar + parses a primitive type as a name, bills it the same way since + #1568: `int`, `boolean`, `void` and the other primitives are + operators keyed by their spelling, as a declared type and as a class + literal's receiver. Groovy reserves these words, so as a member name + or a label (`x.int`, `[int: 1]`) they stay operands. Java's + `boolean` is listed as `boolean` rather than its node kind + `boolean_type`, so both languages report it alike. #1561 made TypeScript's type-level operator keywords operators, as their value-level siblings `as`, `typeof` and `extends` already were: `satisfies`, `keyof`, `infer`, and the `is` and `asserts` of a type predicate. #1563 made TypeScript's diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 2ce3af7c..e98f3340 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -2946,20 +2946,15 @@ mod tests { // place of Java's `MessageFormat.format(...)`. amaanq's // grammar inherits Java's tokenisation, so n1/N1/n2/N2 // shapes match Java up to those substitutions. - // The dekobon grammar parses primitive type names - // (`void`, `int`, `String`) as `type_identifier` - // rather than as distinct keyword tokens, so they - // count as operands here — the prior amaanq grammar - // treated them as operators. Net shift: −2 unique - // operators (`void`, `int`), +2 unique operands - // (`void`, `int` were the only two type_identifiers - // not already counted as operands, since `String` - // was already an identifier in the prior grammar's - // counting). - // `class` is the ninth unique operator since #1552, and - // `static` the tenth since #1565. - assert_eq!(metric.halstead.unique_operators(), 10); - assert_eq!(metric.halstead.unique_operands(), 13); + // The dekobon grammar parses the primitive type names + // `void` and `int` as `type_identifier`, the kind of + // `String`; since #1568 their spelling bills them as + // operators, as Java's keyword tokens are, while + // `String` stays an operand. + // `class` is an operator since #1552, `static` since + // #1565, and `void` and `int` since #1568. + assert_eq!(metric.halstead.unique_operators(), 12); + assert_eq!(metric.halstead.unique_operands(), 11); // `bugs` is `effort.powf(2/3) / 3000`, and the platform libm's // `powf` differs in the last ulp: macOS rendered this value // as `…92342184` where Linux and Windows gave `…923421845`. @@ -2969,20 +2964,20 @@ mod tests { { ".bugs" => insta::rounded_redaction(12) }, @r#" { - "unique_operators": 10, - "total_operators": 24, - "unique_operands": 13, - "total_operands": 23, + "unique_operators": 12, + "total_operators": 26, + "unique_operands": 11, + "total_operands": 21, "length": 47, - "estimated_program_length": 81.32499728470782, - "purity_ratio": 1.730319091163996, + "estimated_program_length": 81.07329781366414, + "purity_ratio": 1.7249637832694498, "vocabulary": 23, "volume": 212.60741193467962, - "difficulty": 8.846153846153847, - "level": 0.11304347826086956, - "effort": 1880.7578748067813, - "time": 104.48654860037674, - "bugs": 0.050788728209 + "difficulty": 11.454545454545455, + "level": 0.0873015873015873, + "effort": 2435.3212639790577, + "time": 135.29562577661432, + "bugs": 0.06033688972 } "# ); @@ -3009,36 +3004,38 @@ mod tests { |metric| { // The dekobon grammar consolidates the 8 primitive // type names (`byte`, `short`, `int`, `long`, `char`, - // `float`, `double`, `boolean`) under `type_identifier` - // — so they count as operands, not as distinct - // operators. Likewise numeric literals collapse to one - // `NumberLiteral` shape (no Hex/Octal/Binary/Decimal - // split), and `'x'` parses as `StringLiteral` (Groovy - // single-quoted strings) rather than as - // `CharacterLiteral`. Operators remaining in this - // fixture: `=`, the `class` keyword (#1552) and the - // class-body braces (only `{` is in the operator set). - // True/false collapse under one `BooleanLiteral`. - assert_eq!(metric.halstead.unique_operators(), 3); - assert_eq!(metric.halstead.unique_operands(), 27); + // `float`, `double`, `boolean`) under `type_identifier`; + // since #1568 each bills as a distinct operator keyed by + // its text, as Java's `java_primitive_ops` twin does. + // Numeric literals collapse to one `NumberLiteral` shape + // (no Hex/Octal/Binary/Decimal split), and `'x'` parses + // as `StringLiteral` (Groovy single-quoted strings) + // rather than as `CharacterLiteral`. + // expected: operators `=` x9, `class`, `{}` and the eight + // primitives (`boolean` twice) — n1 11, N1 20; operands + // `Prims`, the nine names and nine values — n2 19, N2 19. + assert_eq!(metric.halstead.unique_operators(), 11); + assert_eq!(metric.halstead.total_operators(), 20); + assert_eq!(metric.halstead.unique_operands(), 19); + assert_eq!(metric.halstead.total_operands(), 19); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 3, - "total_operators": 11, - "unique_operands": 27, - "total_operands": 28, + "unique_operators": 11, + "total_operators": 20, + "unique_operands": 19, + "total_operands": 19, "length": 39, - "estimated_program_length": 133.1368500605771, - "purity_ratio": 3.413765386168644, + "estimated_program_length": 118.76437056043838, + "purity_ratio": 3.045240270780471, "vocabulary": 30, "volume": 191.36873322873222, - "difficulty": 1.5555555555555556, - "level": 0.6428571428571428, - "effort": 297.6846961335835, - "time": 16.53803867408797, - "bugs": 0.014861058740943886 + "difficulty": 5.5, + "level": 0.18181818181818182, + "effort": 1052.5280327580272, + "time": 58.47377959766818, + "bugs": 0.0344906352248725 } "# ); @@ -3123,10 +3120,13 @@ mod tests { #[cfg(feature = "groovy")] #[test] fn groovy_closure_operators_and_operands() { - check_metrics::("def double = { x -> x * 2 }", "foo.groovy", |metric| { + // `twice`, not the original `double`: Groovy reserves the + // primitive type names, so `def double` is not valid Groovy and + // `double` would bill as the primitive operator (#1568). + check_metrics::("def twice = { x -> x * 2 }", "foo.groovy", |metric| { // Closure with arrow-style parameter list. // Distinct operators: def, =, {}, ->, * = 5. - // Distinct operands: double, x, 2 = 3. + // Distinct operands: twice, x, 2 = 3. assert_eq!(metric.halstead.unique_operators(), 5); assert_eq!(metric.halstead.unique_operands(), 3); }); @@ -6304,7 +6304,9 @@ end", /// #1566: the class-literal shapes Java and Groovy spell alike. The /// expected parts are what Groovy — untouched by #1566 — already /// billed for `Foo.class`: the type name an operand, the `.` and - /// `[]` operators, and the `class` tail an operand. Both tests below + /// `[]` operators, and the `class` tail an operand. A primitive or + /// `void` receiver is an operator keyed by its spelling, as Java's + /// keyword is; Groovy joined that half in #1568. Both tests below /// read this one table, so the two languages cannot drift apart. #[cfg(any(feature = "groovy", feature = "java"))] const SHARED_CLASS_LITERALS: &[ClassLiteralParts] = &[ @@ -6315,6 +6317,9 @@ end", &["."], &["List", "class", "java", "util"], ), + ("int.class", &[".", "int"], &["class"]), + ("int[].class", &[".", "[]", "int"], &["class"]), + ("void.class", &[".", "void"], &["class"]), ]; /// Asserts each literal's operator and operand vocabularies inside @@ -6358,8 +6363,8 @@ end", /// already made. Before the fix the literal billed as one operand /// spelled by its whole text while `.`, `[]` and a primitive inside /// it also billed as operators. The shared rows take their expected - /// values from Groovy; the primitive and `void` rows are Java's own, - /// since Groovy still bills `int` and `void` as operands (#1568). + /// values from Groovy, whose primitive and `void` rows joined them + /// in #1568. #[cfg(feature = "java")] #[test] fn java_class_literal_bills_its_parts_1566() { @@ -6373,16 +6378,6 @@ end", FRAME_OPERANDS, SHARED_CLASS_LITERALS, ); - assert_class_literal_parts::( - "Foo.java", - frame, - FRAME_OPS, - FRAME_OPERANDS, - &[ - ("int.class", &[".", "int"], &["class"]), - ("void.class", &[".", "void"], &["class"]), - ], - ); // The totals pin the multiplicities a vocabulary cannot: the frame // bills four operators (`class`, `{}`, `=`, `;`) and three operands // (`C`, `Object`, `c`), and each literal adds its parts once — @@ -6417,6 +6412,97 @@ end", ); } + /// #1568: a Groovy primitive type bills as the operator Java's + /// keyword does, keyed by its spelling, in every position the + /// grammar gives it: a `type_identifier` where a type is declared, + /// a plain `identifier` where the grammar misreads an uninitialised + /// local (`int t` parses as two expression statements). Each row's + /// operators are what the Java twin of the issue fixture bills, which + /// `groovy_primitive_type_matches_java_1568` checks directly. + #[cfg(feature = "groovy")] + #[test] + fn groovy_primitive_types_are_operators_1568() { + const ROWS: &[(&str, &[&str], &[&str])] = &[ + ( + "class A { boolean m() { int x = 1; } }\n", + &["()", ";", "=", "boolean", "class", "int", "{}"], + &["1", "A", "m", "x"], + ), + ( + "void m(long a, char b) {}\n", + &["()", ",", "char", "long", "void", "{}"], + &["a", "b", "m"], + ), + ( + "def m() { short s\n double d }\n", + &["()", "def", "double", "short", "{}"], + &["d", "m", "s"], + ), + ( + "byte[] a = (float) b as byte\n", + &["()", "=", "[]", "as", "byte", "float"], + &["a", "b"], + ), + // A user type's name stays an operand, however close its + // spelling: only the exact keyword bills. + ( + "Integer a\nInt b\ninte c\n", + &[], + &["Int", "Integer", "a", "b", "c", "inte"], + ), + ]; + for (source, operators, operands) in ROWS { + let ops = ops_of::(source, "foo.groovy"); + assert_eq!(ops.operators, *operators, "operators of {source:?}"); + assert_eq!(ops.operands, *operands, "operands of {source:?}"); + } + } + + /// #1568's oracle: the issue's fixture scores the same in Groovy and + /// Java, vocabulary and multiplicities alike. + #[cfg(all(feature = "groovy", feature = "java"))] + #[test] + fn groovy_primitive_type_matches_java_1568() { + let source = "class A { boolean m() { int x = 1; } }\n"; + let groovy = ops_of::(source, "foo.groovy"); + let java = ops_of::(source, "Foo.java"); + assert_eq!(groovy.operators, java.operators); + assert_eq!(groovy.operands, java.operands); + // `()`, `;`, `=`, `boolean`, `class`, `int`, `{}` x2 / `A`, `m`, + // `x`, `1` + assert_halstead_rows::( + "foo.groovy", + &[("issue fixture", source, [8, 4])], + |h| [h.total_operators(), h.total_operands()], + ); + assert_halstead_rows::("Foo.java", &[("issue fixture", source, [8, 4])], |h| { + [h.total_operators(), h.total_operands()] + }); + } + + /// #1568: Groovy reserves the primitive type names, so one can name + /// something only as a member selector after a `.`-family operator + /// or as a map or named-argument label. Each position stays an + /// operand; each row is its own source so a selector kind dropped + /// from the gate fails by name. + #[cfg(feature = "groovy")] + #[test] + fn groovy_primitive_spelled_names_stay_operands_1568() { + for source in [ + "x.int\n", + "x?.int\n", + "x??.int\n", + "x*.int\n", + "x.@int\n", + "x.&int\n", + "x::int\n", + "def m = [int: 1]\n", + "f(int: 1)\n", + ] { + assert_keywords_are_operands_only::(source, "foo.groovy", &["int"]); + } + } + /// #1552: Groovy's type-declaration keywords are operators, as /// Java's are. `String.class` is a `field_access` whose `class` is a /// plain identifier, so it stays an operand with no guard needed. @@ -6432,8 +6518,8 @@ end", ("trait", "trait T {}\n", [2, 2]), // `enum`, `{}` ("enum", "enum E { A }\n", [2, 2]), - // `record`, `()`, `{}` - ("record", "record R(int x) {}\n", [3, 3]), + // `record`, `()`, `int`, `{}` — as Java's row, since #1568 + ("record", "record R(int x) {}\n", [4, 4]), // `def`, `=`, `.` — no `class` ("class reference", "def a = String.class\n", [3, 3]), // `=` twice: a script-level `record` / `trait` used as a name @@ -6586,12 +6672,12 @@ end", [8, 9], ), // `class`, `{}` x3, `native`, `synchronized`, `strictfp`, - // `()` x3 + // `void` x3 (#1568), `()` x3 ( "method modifiers", "class C {\n native void n()\n synchronized void s() {}\n \ strictfp void f() {}\n}\n", - [6, 10], + [7, 13], ), // `sealed`, `class` x2, `permits`, `{}` x2, `non-sealed`, // `extends` @@ -6606,12 +6692,12 @@ end", "sealed interface I permits A {}\nsealed trait T permits A {}\n", [5, 8], ), - // `class`, `{}` x3, `static` x2, `()`: an initializer and a - // script-level method + // `class`, `{}` x3, `static` x2, `void` (#1568), `()`: an + // initializer and a script-level method ( "static initializer and method", "class C {\n static {}\n}\nstatic void top() {}\n", - [4, 7], + [5, 8], ), // `.` x3: no `static`, as `import` bills nothing ( diff --git a/src/ops.rs b/src/ops.rs index 8d66055e..1e193dc8 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -947,19 +947,8 @@ mod tests { // true/false appear as operands; `class` is billed since #1552 // and `public` since #1565. &mut [ - "public", - "class", - "{}", - ";", - "=", - "byte", - "short", - "int", - "long", - "char", - "float", - "double", - "boolean_type", + "public", "class", "{}", ";", "=", "byte", "short", "int", "long", "char", "float", + "double", "boolean", ], &mut [ "Prims", "a", "b", "c", "d", "e", "f", "g", "h", "i", "1", "2", "3", "4", "'x'", @@ -1576,22 +1565,13 @@ mod tests { "foo.java", "class A { List> v; boolean m() { return a < b && c > d; } }\n", &[ - "class", - "<>", - ";", - "{}", - "boolean_type", - "()", - "return", - "<", - "&&", - ">", + "class", "<>", ";", "{}", "boolean", "()", "return", "<", "&&", ">", ], [10, 14], ); } - // expected: as Java, with `bool` for `boolean_type`. + // expected: as Java, with `bool` for `boolean`. #[cfg(feature = "csharp")] #[test] fn csharp_generic_angles_are_one_pair_1559() { @@ -1703,9 +1683,9 @@ mod tests { "foo.groovy", "class A { List> v; def boolean m() { return a < b && c > d } }\n", &[ - "class", "<>", ";", "{}", "def", "()", "return", "<", "&&", ">", + "class", "<>", ";", "{}", "def", "boolean", "()", "return", "<", "&&", ">", ], - [10, 14], + [11, 15], ); } From 18c28caccee5e8118f6ae34ad3e567068789986b Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:07:25 -0700 Subject: [PATCH 07/34] fix(getter/csharp): bill no interpolation clause An interpolated string's alignment `,` and format `:` (`$"{x,8:F2}"`) were billed as Halstead operators. Both delimit the field like the `{` / `}` braces #1314 left unbilled, and Python's format-spec `:` went the same way in #1486. Parent-guard COMMA on interpolation_alignment_clause and COLON on interpolation_format_clause; the same tokens elsewhere, including a ternary, named argument or argument list inside the hole, keep billing. The alignment value still walks as an ordinary expression: `8` is an operand and `{x,-w}` bills `-` and `w`, since the spec makes it a constant_expression that may name a const. The csharp/strings.cs corpus snapshot moves by its two `:F2` colons (Halstead and MI only). Fixes #1573 --- big-code-analysis-ast/src/getter/csharp.rs | 30 +++++++++- big-code-analysis-book/src/metrics.md | 7 ++- src/metrics/halstead.rs | 65 +++++++++++++++++++++ tests/repositories/big-code-analysis-output | 2 +- 4 files changed, 101 insertions(+), 3 deletions(-) diff --git a/big-code-analysis-ast/src/getter/csharp.rs b/big-code-analysis-ast/src/getter/csharp.rs index 4d5f94f3..8d32b22a 100644 --- a/big-code-analysis-ast/src/getter/csharp.rs +++ b/big-code-analysis-ast/src/getter/csharp.rs @@ -49,6 +49,32 @@ fn csharp_comparison_misparse<'a>(list: &Node<'a>, ancestors: Ancestors<'a, '_>) ) } +// Whether `node` is the `,` opening an interpolation's alignment +// (`$"{x,8}"`) or the `:` opening its format string (`$"{x:F2}"`). +// Both delimit the field, as the `{` / `}` braces do, which #1314 left +// unbilled in every interpolating language; Python's format-spec `:` +// went the same way in #1486 (#1573). The guard reads the parent, so a +// ternary `:`, a named-argument `:` or an argument-list `,` written +// inside the hole (`$"{f(a, b)}"`) keeps billing. +// +// The alignment *value* is a `constant_expression` and walks as one: +// `{x,8}` bills `8` and `{x,-W}` bills `-` and `W`. That is where C# +// and Python differ: a Python literal width (`f"{x:>10}"`) is +// format-spec text and bills nothing, while C#'s `8` is an +// `integer_literal` token. An expression width bills in both +// (`f"{x:>{w}}"` bills `w`). The format string after `:` (`F2`) is +// string content and has no node to bill. +fn is_interpolation_clause_delimiter<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let clause = match node.kind_id().into() { + Csharp::COLON => Csharp::InterpolationFormatClause, + Csharp::COMMA => Csharp::InterpolationAlignmentClause, + _ => return false, + }; + ancestors + .parent(node) + .is_some_and(|parent| parent.kind_id() == clause as u16) +} + impl Getter for CsharpCode { fn get_space_kind(node: &Node) -> SpaceKind { use Csharp::*; @@ -94,7 +120,9 @@ impl Getter for CsharpCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Csharp::*; - if GENERIC_ANGLES.is_closer(node, ancestors) { + if GENERIC_ANGLES.is_closer(node, ancestors) + || is_interpolation_clause_delimiter(node, ancestors) + { return TokenRole::Unknown; } diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 22e2a691..78f7a9b2 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -693,7 +693,12 @@ tokens involved the same way they spell real operators: `:` that opens a Python format spec (`f"{x:>10}"`) counts nothing either, because the spec after it is string content (#1486). Nor does the self-documenting `=` of `f"{x=}"`, which prints the - expression's text and assigns nothing (#1551). + expression's text and assigns nothing (#1551). C#'s format `:` and + alignment `,` (`$"{x,8:F2}"`) count nothing for the same reason + (#1573), but the alignment is an expression and keeps its own + vocabulary: `8` is an operand, and `{x,-w}` counts `-` and `w`. A + Python literal width (`f"{x:>10}"`) is format-spec text and counts + nothing. - **A delimiter around *syntax* is an operator, whatever it delimits.** The rule above is about a literal's own punctuation, and it stops there. Halstead counts the vocabulary a program is written in and has diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index e98f3340..775586af 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -3704,6 +3704,71 @@ mod tests { ); } + /// #1573: an interpolation's alignment `,` and format `:` delimit the + /// field, as Python's format-spec `:` does (#1486), and bill nothing. + /// The alignment value is an ordinary expression, so `8`, `Width` + /// and `-w` keep their vocabulary. The guard reads the token's + /// parent: the last seven rows put the same `,` / `:` inside and + /// outside a hole in a position that is not a clause, and each keeps + /// billing. Pre-fix, every row with a clause billed one extra `,` + /// and/or `:`. + /// + /// Each row is `(label, source, operators, operands)`, the two + /// vocabularies space-separated. No row repeats a token, so the + /// expected `[n1, N1, n2, N2]` is the two vocabulary sizes, each twice. + #[cfg(feature = "csharp")] + #[test] + fn csharp_interpolation_clause_delimiters_are_not_operators_1573() { + const ROWS: &[(&str, &str, &str, &str)] = &[ + ("control", "s = $\"{x}\";", "; =", "s x"), + ("format", "s = $\"{x:F2}\";", "; =", "s x"), + ("alignment", "s = $\"{x,8}\";", "; =", "8 s x"), + ("both", "s = $\"{x,8:F2}\";", "; =", "8 s x"), + // a `const` width is a `constant_expression` the spec allows + ("named width", "s = $\"{x,Width}\";", "; =", "Width s x"), + ("negated width", "s = $\"{x,-w}\";", "- ; =", "s w x"), + // the format string's `,` is text, not a token + ("comma in format", "s = $\"{x:#,##0}\";", "; =", "s x"), + ("raw", "s = $$\"\"\"{{x,8:F2}}\"\"\";", "; =", "8 s x"), + ("verbatim", "s = $@\"{x:F2}\";", "; =", "s x"), + ("call in hole", "s = $\"{f(a, b)}\";", "() , ; =", "a b f s"), + ( + "ternary in hole", + "s = $\"{(a ? b : c)}\";", + "() : ; = ?", + "a b c s", + ), + ( + "named arg in hole", + "s = $\"{f(x: 1)}\";", + "() : ; =", + "1 f s x", + ), + ("ternary", "s = a ? b : c;", ": ; = ?", "a b c s"), + ("named argument", "f(x: 1);", "() : ;", "1 f x"), + ( + "case label", + "switch (x) { case 1: break; }", + "() : ; break case switch {}", + "1 x", + ), + ("argument list", "f(a, b);", "() , ;", "a b f"), + ]; + let sorted = |words: &[String]| { + let mut words: Vec<&str> = words.iter().map(String::as_str).collect(); + words.sort_unstable(); + words.join(" ") + }; + for (label, source, operators, operands) in ROWS { + let ops = ops_of::(source, "foo.cs"); + assert_eq!(sorted(&ops.operators), *operators, "{label}: {source:?}"); + assert_eq!(sorted(&ops.operands), *operands, "{label}: {source:?}"); + let n1 = operators.split(' ').count() as u64; + let n2 = operands.split(' ').count() as u64; + assert_halstead_counts::(source, "foo.cs", [n1, n1, n2, n2], label); + } + } + // C# half of #1380 — see `java_self_and_super_references_are_operands` // for the structural argument. `base` moves with `this`: both are // receivers of a member access. diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index 2128d6da..b415bc3b 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit 2128d6dad3f59d5ad973696123ecd99b296e0797 +Subproject commit b415bc3bae8888a032ec99f78f3f2f7529fd0ead From 023038d4cfb10058e37b75bd9f1755e6a8303626 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:26:34 -0700 Subject: [PATCH 08/34] chore(thresholds): re-measure halstead.effort Re-derive the halstead.effort default after the Halstead vocabulary changes (#1486-#1573) on a rebuilt 41-repository, 20-language corpus, measured with this branch's release bca and, for comparison, v2.3.0. At 50,000 the median language flags 2.20% of its functions (v2.3.0: 2.23%), inside the 1-3% sizing band, so the default stays. Update its rationale (per-language p90 now 3k Ruby to 60k C), regenerate the book's per-language halstead.effort column, correct the two book paragraphs the new figures contradict, and record the corpus with commit SHAs in docs/development/threshold_corpus.md so the next re-derivation can reproduce it. Fixes #1569 --- AGENTS.md | 4 +- .../src/recipes/thresholds.md | 56 +++++--- .../src/default_thresholds.rs | 10 +- docs/development/threshold_corpus.md | 123 ++++++++++++++++++ 4 files changed, 169 insertions(+), 24 deletions(-) create mode 100644 docs/development/threshold_corpus.md diff --git a/AGENTS.md b/AGENTS.md index ff2204ea..dbda1dff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -681,7 +681,9 @@ stranded `.snap.new` drift that blocks CI on every subsequent change. The limits `bca init` scaffolds are defined once, in `big-code-analysis-cli/src/default_thresholds.rs`. They are derived from published thresholds plus a 20-language corpus measurement (#1140), -and each carries its rationale inline. Change them there, not in a +and each carries its rationale inline. `halstead.effort` was re-measured +in #1569 on a rebuilt corpus recorded in +[`docs/development/threshold_corpus.md`](docs/development/threshold_corpus.md). Change them there, not in a copy: the scaffolded `bca.toml` renders its `[thresholds]` block from that table, so there is no second copy of the numbers to drift. diff --git a/big-code-analysis-book/src/recipes/thresholds.md b/big-code-analysis-book/src/recipes/thresholds.md index 2b428797..1c68f908 100644 --- a/big-code-analysis-book/src/recipes/thresholds.md +++ b/big-code-analysis-book/src/recipes/thresholds.md @@ -93,31 +93,45 @@ tighten, not evidence that you should. | Language | `cognitive` | `cyclomatic` | `abc` | `halstead.effort` | `loc.ploc` | | --- | --- | --- | --- | --- | --- | -| Bash | 35 | 25 | 60 | 95000 | (thin sample) | -| C | 50 | 30 | 60 | 200000 | 3500 | +| Bash | 35 | 25 | 60 | 44000 | (thin sample) | +| C | 50 | 30 | 60 | 180000 | 3500 | | C++ | 14 | 10 | 25 | 50000 | (thin sample) | -| C# | 4 | 4 | 13 | 9500 | 700 | +| C# | 4 | 4 | 13 | 44000 | 700 | | Elixir | 5 | 7 | 20 | 15000 | 1500 | -| Go | 25 | 17 | 45 | 120000 | 1500 | -| Groovy | 16 | 13 | 25 | 30000 | 450 | -| Java | 7 | 5 | 16 | 15000 | 800 | -| JavaScript | 20 | 15 | 35 | 55000 | (thin sample) | -| Kotlin | 10 | 8 | 20 | 20000 | 300 | -| Lua | 35 | 20 | 40 | 85000 | 800 | -| Objective-C | 18 | 12 | 35 | 60000 | (thin sample) | -| Perl | 35 | 25 | 45 | 140000 | 1500 | -| PHP | 10 | 9 | 25 | 50000 | 900 | -| Python | 20 | 13 | 25 | 25000 | 700 | -| Ruby | 7 | 6 | 19 | 10000 | 400 | -| Rust | 8 | 8 | 20 | 35000 | 900 | -| Tcl | 45 | 25 | 85 | 90000 | (thin sample) | -| TypeScript | 20 | 13 | 25 | 55000 | 450 | -| TSX | 14 | 10 | 65 | 95000 | 400 | +| Go | 25 | 17 | 45 | 89000 | 1500 | +| Groovy | 16 | 13 | 25 | 25000 | 450 | +| Java | 7 | 5 | 16 | 20000 | 800 | +| JavaScript | 20 | 15 | 35 | 170000 | (thin sample) | +| Kotlin | 10 | 8 | 20 | 18000 | 300 | +| Lua | 35 | 20 | 40 | 82000 | 800 | +| Objective-C | 18 | 12 | 35 | 73000 | (thin sample) | +| Perl | 35 | 25 | 45 | 41000 | 1500 | +| PHP | 10 | 9 | 25 | 26000 | 900 | +| Python | 20 | 13 | 25 | 42000 | 700 | +| Ruby | 7 | 6 | 19 | 5000 | 400 | +| Rust | 8 | 8 | 20 | 34000 | 900 | +| Tcl | 45 | 25 | 85 | 46000 | (thin sample) | +| TypeScript | 20 | 13 | 25 | 60000 | 450 | +| TSX | 14 | 10 | 65 | 130000 | 400 | "Thin sample" marks a language whose file count in the corpus is too small to derive a file-size figure from. The per-function figures for Bash, Tcl, JavaScript, and Objective-C rest on the smallest samples in the table and are the ones most worth re-deriving against your own code. +The `halstead.effort` column comes from a different measurement than the rest of the table. It was +re-measured on 2026-10-10, after the Halstead vocabulary changes in +[#1486](https://github.com/dekobon/big-code-analysis/issues/1486) through +[#1573](https://github.com/dekobon/big-code-analysis/issues/1573), on a rebuilt corpus of 41 +repositories that is comparable to the original but not identical to it. The repository list and +commits are recorded in +[`docs/development/threshold_corpus.md`](https://github.com/dekobon/big-code-analysis/blob/main/docs/development/threshold_corpus.md). +Several rows moved by a factor of two to five, and almost all of that movement came from the change +of repositories, not from the metric: the same 41 repositories measured with the pre-change release +differ by at most half (Python) and mostly not at all. Treat any single row as a property of the +repositories that represent the language, which is one more reason to re-derive against your own. +In this column the thinnest samples are Objective-C, Perl, Bash, and C++, each under 2,500 +functions. + ### What to change, and why {#per-language-changes} Most languages need no override. The cases that do fall into three groups. @@ -126,7 +140,8 @@ Most languages need no override. The cases that do fall into three groups. to three times the defaults. These languages lack exceptions or discourage them, so error handling is inline branching, and they favour long `switch`-style dispatch over polymorphism. The default `cognitive` limit flags 5% to 15% of their functions, against 3% in the median language. Raise -`cognitive` and `halstead.effort` first; those two carry most of the excess. +`cognitive` first. `halstead.effort` carries the rest of the excess for C, Go, and Lua; Tcl, Bash, +and Perl measure close to the default on it. In a single-language repository this is the whole `[thresholds]` table; in a mixed one it belongs under the language it describes, or it drags every other language up with it. @@ -160,7 +175,8 @@ wmc = 40 ``` **Compact, heavily-abstracted languages.** Java, Ruby, Kotlin, C#, and Elixir sit far under the -defaults on every per-function metric. Their 97.5th percentile `cyclomatic` is 4 to 8. Gating them +defaults on every per-function metric except C#'s `halstead.effort`. Their 97.5th percentile +`cyclomatic` is 4 to 8. Gating them at 15 means the gate will effectively never fire, so tighten if you want it to do work. Read the C# row with care. Its distribution is dominated by property accessors, which `bca` counts diff --git a/big-code-analysis-cli/src/default_thresholds.rs b/big-code-analysis-cli/src/default_thresholds.rs index 5c2bbb39..47e1be92 100644 --- a/big-code-analysis-cli/src/default_thresholds.rs +++ b/big-code-analysis-cli/src/default_thresholds.rs @@ -15,6 +15,9 @@ //! and checked against a measured corpus: 43 real-world repositories //! across 20 languages (tests, vendored trees, and generated code //! excluded), 254k function spaces, 40k container spaces, 27k files. +//! `halstead.effort` alone was re-measured after the #1486-#1573 +//! vocabulary changes, on a rebuilt 41-repository corpus recorded in +//! `docs/development/threshold_corpus.md` (#1569). //! The design rule is that a default should flag genuine outliers — //! roughly 1-3% of in-scope spaces in the median language — because a //! gate that fires on a tenth of a codebase is a style rule, not a @@ -102,9 +105,10 @@ pub(crate) const DEFAULT_THRESHOLDS: &[DefaultThreshold] = &[ limit: 50_000, rationale: &[ "No published threshold exists for effort, so this one is purely", - "percentile-derived. It is also the most language-dependent limit", - "in the table — median-language p90 effort ranges from 2k (Java)", - "to 56k (C) — so it is the first one worth overriding per language.", + "percentile-derived: it fires on ~2% of functions in the median", + "language. It is also the most language-dependent limit in the", + "table — per-language p90 effort ranges from 3k (Ruby) to 60k", + "(C) — so it is the first one worth overriding per language.", ], }, DefaultThreshold { diff --git a/docs/development/threshold_corpus.md b/docs/development/threshold_corpus.md new file mode 100644 index 00000000..76f4caa7 --- /dev/null +++ b/docs/development/threshold_corpus.md @@ -0,0 +1,123 @@ +# Threshold calibration corpus + +This file records the corpus behind the `halstead.effort` default in +[`big-code-analysis-cli/src/default_thresholds.rs`](../../big-code-analysis-cli/src/default_thresholds.rs) +and the `halstead.effort` column of the book's per-language table +([Choosing thresholds](../../big-code-analysis-book/src/recipes/thresholds.md#per-language)). +The other defaults and columns still rest on +[#1140](https://github.com/dekobon/big-code-analysis/issues/1140)'s 2026-07-31 corpus: 43 repositories +across 20 languages, whose repository list was never recorded. + +The corpus below is a **rebuilt** one, made for +[#1569](https://github.com/dekobon/big-code-analysis/issues/1569) after the Halstead vocabulary +changes (#1486 through #1573). It covers the same 20 languages with the same sizing rule and the same kinds of +exclusions, so it is comparable to #1140's, but it is not the same set of repositories. + +## Method + +- Measured 2026-10-10 with a release `bca` built from `fix/batch-2026-10-10` at `18c28cac`, with + every vocabulary change applied. The same corpus was also measured with the `v2.3.0` release, which + predates all of them, to separate the metric change from the corpus change. +- Each repository was cloned with `git clone --depth 1` at its default branch. +- `bca metrics --no-config -O json -p ` ran once per repository. The `-I` globs restricted it to + the language's own extensions: `*.sh` (Bash-it also `*.bash`), `*.c`, `*.cpp *.cc *.h *.hpp`, `*.cs`, + `*.ex`, `*.go`, `*.groovy`, `*.java`, `*.js *.mjs`, `*.kt`, `*.lua`, `*.m`, `*.pm *.pl`, `*.php`, + `*.py`, `*.rb`, `*.rs`, `*.tcl`, `*.ts`, `*.tsx`. C counts `.c` only, because `bca` parses `.h` with + the C++ grammar. +- The run excluded test, vendored, generated, and documentation-demo trees with these `-X` globs: + `**/test/**`, `**/tests/**`, `**/testing/**`, `**/__tests__/**`, `**/spec/**`, `**/specs/**`, + `**/t/**`, `**/testdata/**`, `**/fixtures/**`, `**/*_test.*`, `**/*_tests.*`, `**/test_*.py`, + `**/*Test.java`, `**/*Tests.java`, `**/*Test.kt`, `**/*Test.groovy`, `**/*Spec.groovy`, + `**/*.test.*`, `**/*.spec.*`, `**/*_spec.rb`, `**/*Tests/**`, `**/*.Tests/**`, `**/*.Test/**`, + `**/*.Tests.*/**`, `**/benches/**`, `**/benchmark/**`, `**/benchmarks/**`, `**/vendor/**`, + `**/third_party/**`, `**/thirdparty/**`, `**/third-party/**`, `**/node_modules/**`, `**/deps/**`, + `**/dist/**`, `**/build/**`, `**/generated/**`, `**/*.min.js`, `**/*.d.ts`, `**/*.generated.*`, + `**/*.demo.*`, `**/*.story.*`, `**/*.stories.*`, `**/demos/**`, `**/*-tests/**`, `**/*.pb.go`, + `**/*_string.go`, `**/*.Designer.cs`, `**/*.designer.cs`. + The per-repository extras in the corpus table exclude a duplicate source tree (Guava's `android/` + backport), vendored libraries (three.js `libs/`), generated validators (webpack), and sample apps. + Files `bca` recognises as generated are skipped on top of this. +- Values are each `function`-kind space's own `metrics.halstead.effort`, walked through nested spaces. + That is the space kind `halstead.effort`'s `Function` scope admits in `src/metric_catalog.rs`. + Percentiles use linear interpolation between closest ranks. + +## Result + +| Language | Functions | p50 | p90 | p97.5 | % > 50,000 | p97.5 at v2.3.0 | % > 50,000 at v2.3.0 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Bash | 1,803 | 803 | 8,699 | 44,195 | 2.1 | 44,195 | 2.1 | +| C | 6,361 | 5,582 | 60,359 | 182,534 | 11.8 | 173,763 | 11.3 | +| C# | 6,214 | 375 | 6,914 | 43,715 | 2.3 | 43,957 | 2.3 | +| C++ | 2,491 | 600 | 12,005 | 49,665 | 2.5 | 49,790 | 2.4 | +| Elixir | 4,535 | 460 | 4,398 | 15,121 | 0.4 | 15,121 | 0.4 | +| Go | 9,454 | 1,310 | 21,802 | 89,276 | 4.7 | 89,276 | 4.7 | +| Groovy | 12,529 | 221 | 5,275 | 25,353 | 1.4 | 23,038 | 1.3 | +| Java | 18,198 | 523 | 5,166 | 19,870 | 0.8 | 16,936 | 0.6 | +| JavaScript | 31,030 | 1,191 | 32,475 | 174,060 | 7.2 | 174,108 | 7.3 | +| Kotlin | 6,682 | 428 | 4,945 | 18,334 | 0.6 | 17,413 | 0.6 | +| Lua | 4,562 | 2,673 | 27,416 | 81,544 | 5.1 | 81,544 | 5.1 | +| Objective-C | 1,209 | 1,215 | 15,441 | 72,897 | 4.1 | 75,726 | 4.1 | +| Perl | 1,760 | 640 | 12,537 | 41,205 | 2.0 | 43,862 | 2.2 | +| PHP | 20,296 | 625 | 6,857 | 26,333 | 1.2 | 26,333 | 1.2 | +| Python | 10,264 | 1,030 | 10,904 | 41,983 | 2.0 | 28,535 | 1.2 | +| Ruby | 11,837 | 453 | 2,774 | 4,974 | 0.0 | 4,974 | 0.0 | +| Rust | 8,778 | 998 | 10,472 | 33,894 | 1.4 | 33,309 | 1.3 | +| Tcl | 10,907 | 804 | 13,412 | 45,543 | 2.3 | 45,537 | 2.3 | +| TSX | 6,456 | 774 | 26,726 | 134,616 | 6.0 | 131,020 | 6.0 | +| TypeScript | 6,945 | 830 | 14,724 | 60,339 | 3.1 | 60,126 | 3.0 | + +At the shipped limit of 50,000, the median language flags 2.20% of its functions, inside the 1–3% +band the sizing rule asks for, so the default stands. Candidate limits for comparison: 40,000 gives +2.78% and 60,000 gives 1.76%. The `v2.3.0` binary gives 2.23% on the same corpus. The vocabulary +changes therefore barely moved the default's footing. Their largest per-language effect is Python +(p97.5 28,535 → 41,983), followed by Java (+17%) and Groovy (+10%). Bash, Elixir, Go, Lua, PHP, and +Ruby did not move at all, and Tcl moved by under 0.1%. + +Ruby's low figures are a property of these two repositories, which hold their own methods short +with RuboCop's method-length and ABC cops. They are not a measurement gap. + +## Corpus + +| Language | Repository | Commit | Files | Functions | Extra excludes | +| --- | --- | --- | --- | --- | --- | +| Bash | [nvm-sh/nvm](https://github.com/nvm-sh/nvm) | `b1b3a0038c0666c7436cb5d383608b4b72c8300b` | 4 | 154 | | +| Bash | [ohmybash/oh-my-bash](https://github.com/ohmybash/oh-my-bash) | `abf846186ab0a8a41ec5888e827ece6277dfe446` | 218 | 765 | | +| Bash | [Bash-it/bash-it](https://github.com/Bash-it/bash-it) | `81929690975f9a7ea3c50b87265f2959327bb0a5` | 334 | 884 | | +| C | [curl/curl](https://github.com/curl/curl) | `b8f71d6e1199701352c498467c17847237d1c43e` | 372 | 4,718 | | +| C | [libuv/libuv](https://github.com/libuv/libuv) | `4080d7d965fdf291f37ec72677fc5a2ace8fa361` | 120 | 1,643 | | +| C++ | [fmtlib/fmt](https://github.com/fmtlib/fmt) | `35c58f084e7cd79b51997439cff68346f4c724c0` | 20 | 1,426 | | +| C++ | [google/leveldb](https://github.com/google/leveldb) | `bb74ef739973a70ca9f0d90788a1da89b788c0ad` | 98 | 1,065 | | +| C# | [JamesNK/Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) | `52fa3aef1f2cadcd3a3f874251eddc98d3efbbaa` | 243 | 4,129 | | +| C# | [AutoMapper/AutoMapper](https://github.com/AutoMapper/AutoMapper) | `6e8697bc44f02fb54ef6a556a7d134e63485299e` | 92 | 2,085 | | +| Elixir | [phoenixframework/phoenix](https://github.com/phoenixframework/phoenix) | `9f2c1409ca8bb9d55ca5b9cb7b4244ff645fc2d3` | 85 | 1,688 | | +| Elixir | [elixir-ecto/ecto](https://github.com/elixir-ecto/ecto) | `94d69279c517347ff0962b138f4ccd0556486ae2` | 60 | 2,847 | | +| Go | [caddyserver/caddy](https://github.com/caddyserver/caddy) | `d4165a4eb1fdb129e8ce8390099021a00239ecf7` | 225 | 2,603 | | +| Go | [gohugoio/hugo](https://github.com/gohugoio/hugo) | `0732eadd5aead94d4bd4674d87e9bfc79aaf10a7` | 511 | 6,851 | | +| Groovy | [nextflow-io/nextflow](https://github.com/nextflow-io/nextflow) | `ca258a4e9bce2763001dc67770e2bd957ccf1034` | 967 | 9,815 | | +| Groovy | [apache/groovy](https://github.com/apache/groovy) | `000449404bb0fa255c8bd55cbb35e1c3c7edd379` | 399 | 2,714 | | +| Java | [apache/commons-lang](https://github.com/apache/commons-lang) | `dd2c9e85784b288c7507c3736818040b3b457842` | 264 | 4,550 | | +| Java | [google/guava](https://github.com/google/guava) | `6b3a8d7498a7fde122967ba37a023ba3dcb47792` | 666 | 13,648 | `**/android/**` | +| JavaScript | [mrdoob/three.js](https://github.com/mrdoob/three.js) | `11738a9d55e502fb484cefffcea3a66dc44e9fe9` | 1,403 | 14,891 | `**/libs/**` | +| JavaScript | [webpack/webpack](https://github.com/webpack/webpack) | `5ba5dba8b82a0ebd339a5c2be56166400de3460d` | 1,297 | 16,139 | `**/*.check.js` | +| Kotlin | [Kotlin/kotlinx.coroutines](https://github.com/Kotlin/kotlinx.coroutines) | `bd2e9a1b90400fb7b2fa4f8731e7d8148799ab73` | 342 | 3,029 | | +| Kotlin | [square/okhttp](https://github.com/square/okhttp) | `42e5888d45623662c138ed27e80ea28bbe0aa5cc` | 367 | 3,653 | | +| Lua | [Kong/kong](https://github.com/Kong/kong) | `2d2e67ea7a90cc9d81c53177e9fdbe61bd142236` | 609 | 3,722 | | +| Lua | [luarocks/luarocks](https://github.com/luarocks/luarocks) | `a422d213b38c488472ec3bec70f372bb63784cdb` | 96 | 840 | | +| Objective-C | [SDWebImage/SDWebImage](https://github.com/SDWebImage/SDWebImage) | `c3ad5e1a9bf55c9b76d4c362430b5fcded96c502` | 73 | 889 | `**/Examples/**` | +| Objective-C | [CocoaLumberjack/CocoaLumberjack](https://github.com/CocoaLumberjack/CocoaLumberjack) | `b75d3c3198a2930b4491b04fd47cb504b359c750` | 22 | 320 | `**/Demos/**` | +| Perl | [mojolicious/mojo](https://github.com/mojolicious/mojo) | `da40b9c04e31a530b05ce285fca322e135b3b780` | 124 | 1,569 | | +| Perl | [beyondgrep/ack3](https://github.com/beyondgrep/ack3) | `11d698f9407c6bd5dfcff1c264bf0aa60abacc5d` | 27 | 191 | | +| PHP | [laravel/framework](https://github.com/laravel/framework) | `73a25a4ad3d1da83c3c62a100b8902da380015ef` | 1,820 | 17,242 | | +| PHP | [composer/composer](https://github.com/composer/composer) | `43f74517021a59ed4054a801c1472ef66355c49a` | 315 | 3,054 | | +| Python | [django/django](https://github.com/django/django) | `dab0a5c47f159eb58472110dcbc10bd4a9b01adb` | 763 | 9,324 | | +| Python | [Textualize/rich](https://github.com/Textualize/rich) | `9d8f9a372cc5916fd4781fec207ced7ddac2f08f` | 143 | 940 | | +| Ruby | [rubocop/rubocop](https://github.com/rubocop/rubocop) | `6fa180158e70aaa390fd9f8990b273c5bd38f9b8` | 958 | 10,548 | | +| Ruby | [jekyll/jekyll](https://github.com/jekyll/jekyll) | `541d8b2ee75c8907de4744d4b87a7a6f1f997cae` | 97 | 1,289 | | +| Rust | [BurntSushi/ripgrep](https://github.com/BurntSushi/ripgrep) | `3fce3b5bb0236da2df6d99672afb8a719642eca7` | 91 | 3,219 | | +| Rust | [tokio-rs/tokio](https://github.com/tokio-rs/tokio) | `09a57c27ea64bb546fe16822b9d2e65c599f0fb4` | 488 | 5,559 | | +| Tcl | [tcltk/tcllib](https://github.com/tcltk/tcllib) | `48b8b5dbdf23ced85530a8a2fe8aab1f4bf09f34` | 770 | 7,510 | | +| Tcl | [tcltk/tklib](https://github.com/tcltk/tklib) | `b9f737163cd02f5b243df6c3342fa26e710ab898` | 445 | 3,397 | | +| TypeScript | [vuejs/core](https://github.com/vuejs/core) | `4ab865a848a1da3d10fb674f857e5fff13094644` | 267 | 2,650 | | +| TypeScript | [nestjs/nest](https://github.com/nestjs/nest) | `2731c18054fb348779311d81e4e20949e313be78` | 1,071 | 4,295 | `**/sample/**` | +| TSX | [excalidraw/excalidraw](https://github.com/excalidraw/excalidraw) | `e65eca999e071503efb893d3379a29fb96ff37c4` | 233 | 2,575 | | +| TSX | [mantinedev/mantine](https://github.com/mantinedev/mantine) | `f7ab1ef52579366e2310fc12b1a12e88a821053a` | 759 | 3,881 | | From 20cc3b5e7def460f0fc4e27bcb53b2295902e0db Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:33:36 -0700 Subject: [PATCH 09/34] docs(changelog): record the Halstead vocabulary batch Add the Unreleased entries for #1563-#1569 and #1573. --- CHANGELOG.md | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c00dafa0..1aec70ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,71 @@ for historical reference. ## [Unreleased] +### Changed + +- **Halstead bills angle brackets that enclose types as one `<>` + operator**, like a generic list's (#1567). Rust qualified paths + (`::f`, `::f`), C# function pointer types + (`delegate*`) and Kotlin's qualified `super` each + billed the comparison `<` and `>` before. `N1` falls by one per + occurrence; `n1` changes where a comparison `<` or `>` remains beside + the new `<>`. Kotlin's `id(a)` (#1394) and the Rust macro token + tree stay exceptions. +- **Java class literals bill their parts instead of one operand spelled + by the whole literal** (#1566). JLS §15.8.2 defines `Foo.class` as an + expression, not a literal, so `java.util.List.class` bills three `.` + operators plus the operands `java`, `util`, `List` and `class`; + `Foo[].class` bills `[]`, `.`, `Foo` and `class`; `int.class` bills + `int`, `.` and `class`. Groovy's `Foo.class` and Kotlin's and PHP's + `Foo::class` already scored this way. Before, the literal's `.`, `[]` + and primitive also billed beside the whole-text operand, so each + literal paid for part of its text twice. +- **`bca ops` lists Java's `boolean` as `boolean` instead of its node + kind `boolean_type`** (#1568), so Java and Groovy report the same + vocabulary for identical source. Java's Halstead counts are + unchanged. +- **The `halstead.effort` default was re-measured after the vocabulary + changes** (#1569), on a rebuilt 41-repository, 20-language corpus. It + stays at 50,000, flagging 2.2% of functions in the median language. + The book's per-language `halstead.effort` column is regenerated, and + the corpus is recorded with commit SHAs in + `docs/development/threshold_corpus.md` so it can be reproduced. + +### Fixed + +- **TypeScript and TSX bill their member modifiers as operators** + (#1563): `public`, `private`, `protected`, `override`, `readonly`, + `abstract` and `accessor`, as C# and Kotlin do, counting each keyword + once and never its `accessibility_modifier` / `override_modifier` + wrapper. JavaScript, TypeScript and TSX bill `static`. The same words + used as names (`let readonly = 1`, `obj.static`) stay operands. +- **Kotlin bills `typealias`, `init`, `where`, `by`, `get` and `set` as + operators** (#1564), as C# bills `get`, `set`, `init` and `where`. + Each counts once, as the keyword and never as its wrapper node. The + `@get:` / `@set:` annotation targets count nothing, like `@field:`. + The words used as names (`val by = 1`, `x.get(i)`) and the accessor's + backing `field` stay operands. +- **Java and Groovy bill every modifier as an operator** (#1565), as C# + and Kotlin do. Before, only `abstract`, `final`, `transient`, + `synchronized` and `default` billed. Now `public`, `private`, + `protected`, `static`, `native`, `strictfp`, `volatile`, `sealed`, + `non-sealed` and the `permits` clause keyword bill as well, each as + its keyword and never as Java's `modifiers` wrapper. The `static` of + `import static` and `requires static` bills nothing, as `import` does + not, and the words used as names (`var sealed = 1`, Groovy's + `[static: 1]`) stay operands. +- **Groovy bills the primitive type names as operators keyed by their + spelling** (#1568): `boolean`, `byte`, `char`, `double`, `float`, + `int`, `long`, `short` and `void`, as Java does, covering declared + types, class-literal receivers (`int.class`) and uninitialised locals. + Where Groovy allows the reserved word as a name (`x.int`, `x?.int`, + `x::int`, `[int: 1]`, `f(int: 1)`) it stays an operand. +- **C# no longer bills an interpolated string's alignment `,` or format + `:`** (`$"{x,8:F2}"`) as operators (#1573), matching Python's + format-spec `:`. The alignment value still counts: `8` is an operand, + and `{x,-w}` counts `-` and `w`. A `,` or `:` inside the hole — a call + argument list, ternary or named argument — still counts. + ## [2.3.1] - 2026-10-10 For the `big-code-analysis` crate this release changes metric values From 987b97e7dc844baa5440f59612cb281562395d3d Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:48:46 -0700 Subject: [PATCH 10/34] docs(halstead): apply the batch review findings Correct the CHANGELOG's #1567 n1 claim and name Mozjs and its one-token `static get` under #1563. In the book, add `default` to the modifiers Java and Groovy billed before #1565, document the `static get` newline spelling, and scope the procedural-language threshold advice to cognitive/cyclomatic, since Tcl, Bash and Perl effort sits near the default (AGENTS.md likewise). State `Checker::is_primitive`'s precondition: it is sound only for a node the getter already billed as an operator, which Groovy's kind-only impl relies on. Note that invalid C# interpolations (`$"{x,}"`, `$"{:F2}"`) bill their delimiter, untested by design. Fix the Groovy #1565 test doc that still called `void` an operand, add a Java-to-Groovy twin oracle for #1565, and compare the #1573 test's vocabulary as returned instead of re-sorting it. Re-wrap ragged comments. --- AGENTS.md | 4 +- CHANGELOG.md | 12 ++-- big-code-analysis-ast/src/checker.rs | 8 +++ big-code-analysis-ast/src/getter.rs | 9 +-- big-code-analysis-ast/src/getter/csharp.rs | 5 ++ .../src/getter/generic_angles.rs | 13 ++-- big-code-analysis-ast/src/getter/rust.rs | 9 +-- big-code-analysis-book/src/metrics.md | 60 ++++++++++--------- .../src/recipes/thresholds.md | 14 +++-- src/metrics/halstead.rs | 44 ++++++++++---- 10 files changed, 111 insertions(+), 67 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index dbda1dff..e7e26862 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -711,8 +711,8 @@ Three things to know before quoting a number at anyone: - **These are language-agnostic defaults, and no language is agnostic.** Measured 97.5th-percentile `cognitive` runs from 4 (C#) to 50 (C). C, Tcl, Bash, Lua, Perl, and Go want roughly double the - defaults; Java, Ruby, Kotlin, C#, and Elixir sit far under them. The - per-language table lives in + `cognitive` default; Java, Ruby, Kotlin, C#, and Elixir sit far + under it. The per-language table lives in [Choosing thresholds](big-code-analysis-book/src/recipes/thresholds.md). - **The right limit depends on the job.** A blocking CI gate, an agent edit loop, a legacy triage pass, and a safety-critical build want diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aec70ed..81a57921 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,9 +31,9 @@ for historical reference. (`::f`, `::f`), C# function pointer types (`delegate*`) and Kotlin's qualified `super` each billed the comparison `<` and `>` before. `N1` falls by one per - occurrence; `n1` changes where a comparison `<` or `>` remains beside - the new `<>`. Kotlin's `id(a)` (#1394) and the Rust macro token - tree stay exceptions. + occurrence; `n1` loses any `<` or `>` that no comparison still uses, + and gains `<>` unless `<>` is already present. Kotlin's `id(a)` + (#1394) and the Rust macro token tree stay exceptions. - **Java class literals bill their parts instead of one operand spelled by the whole literal** (#1566). JLS §15.8.2 defines `Foo.class` as an expression, not a literal, so `java.util.List.class` bills three `.` @@ -60,8 +60,10 @@ for historical reference. (#1563): `public`, `private`, `protected`, `override`, `readonly`, `abstract` and `accessor`, as C# and Kotlin do, counting each keyword once and never its `accessibility_modifier` / `override_modifier` - wrapper. JavaScript, TypeScript and TSX bill `static`. The same words - used as names (`let readonly = 1`, `obj.static`) stay operands. + wrapper. JavaScript, Mozjs, TypeScript and TSX bill `static`; the + JavaScript and Mozjs one-token `static get` (a `static get` followed + by a newline) bills as one operator. The same words used as names + (`let readonly = 1`, `obj.static`) stay operands. - **Kotlin bills `typealias`, `init`, `where`, `by`, `get` and `set` as operators** (#1564), as C# bills `get`, `set`, `init` and `where`. Each counts once, as the keyword and never as its wrapper node. The diff --git a/big-code-analysis-ast/src/checker.rs b/big-code-analysis-ast/src/checker.rs index 02e3abe9..00159000 100644 --- a/big-code-analysis-ast/src/checker.rs +++ b/big-code-analysis-ast/src/checker.rs @@ -611,6 +611,14 @@ pub trait Checker { } /// Whether `node` is a primitive type name (`int`, `double`, …), /// which Halstead keys by text so each distinct primitive counts once. + /// + /// Precondition: call this only for a node + /// [`Getter::get_op_type_with_code`] has already classified as + /// [`TokenRole::Operator`]. An impl that cannot read the source may + /// answer by kind alone: Groovy's returns `true` for every + /// `identifier` and `type_identifier`, which is right only because + /// its getter bills those kinds as operators for a primitive + /// spelling and nothing else. #[inline] #[must_use] fn is_primitive(_node: &Node) -> bool { diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index e935c063..1b2ba231 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -232,10 +232,11 @@ pub trait Getter { /// counts the alphabet a program is written in and has no notion of /// a decision, so a `<` is the same vocabulary entry whether it /// opens a JSX tag, brackets a Lua `` attribute, names a C# - /// `operator <` overload, or delimits a Perl `` readline. Every impl that classifies - /// punctuation at all bills `(`, `{`, `[`, `,`, `;`, `.` and `:` on - /// exactly that reading, for each glyph its language spells. Python - /// and Rust were the last two exceptions, closed by #1486. + /// `operator <` overload, or delimits a Perl `` readline. Every + /// impl that classifies punctuation at all bills `(`, `{`, `[`, `,`, + /// `;`, `.` and `:` on exactly that reading, for each glyph its + /// language spells. Python and Rust were the last two exceptions, + /// closed by #1486. /// /// A *decision* metric gates the same token by role, and is /// supposed to disagree: ABC's `conditions` counts a `<` only under diff --git a/big-code-analysis-ast/src/getter/csharp.rs b/big-code-analysis-ast/src/getter/csharp.rs index 8d32b22a..adf1b8b6 100644 --- a/big-code-analysis-ast/src/getter/csharp.rs +++ b/big-code-analysis-ast/src/getter/csharp.rs @@ -64,6 +64,11 @@ fn csharp_comparison_misparse<'a>(list: &Node<'a>, ancestors: Ancestors<'a, '_>) // `integer_literal` token. An expression width bills in both // (`f"{x:>{w}}"` bills `w`). The format string after `:` (`F2`) is // string content and has no node to bill. +// +// On invalid input with no expression before the delimiter +// (`$"{x,}"`, `$"{:F2}"`) error recovery leaves the `,` / `:` outside +// any clause node, so it bills. Valid C# always has both, so this is +// documented rather than tested (grammar-dispatch §6). fn is_interpolation_clause_delimiter<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { let clause = match node.kind_id().into() { Csharp::COLON => Csharp::InterpolationFormatClause, diff --git a/big-code-analysis-ast/src/getter/generic_angles.rs b/big-code-analysis-ast/src/getter/generic_angles.rs index 60e6afb2..97eba85a 100644 --- a/big-code-analysis-ast/src/getter/generic_angles.rs +++ b/big-code-analysis-ast/src/getter/generic_angles.rs @@ -13,12 +13,13 @@ pub(crate) const ANGLE_PAIR: &str = "<>"; /// nodes whose `<` / `>` enclose types rather than compare, and the /// bracket tokens themselves. /// -/// Angle brackets that enclose types are one bracket pair, billed like `()`, `[]` and -/// `{}` (#1395): the opener carries the pair as [`ANGLE_PAIR`] and the -/// closer bills nothing (#1559). The grammars give the brackets the -/// same kind ids as the comparison operators, so only the parent tells -/// the two apart, and the opener is keyed by text rather than by kind -/// to keep it out of the comparison `<`'s vocabulary entry. +/// Angle brackets that enclose types are one bracket pair, billed like +/// `()`, `[]` and `{}` (#1395): the opener carries the pair as +/// [`ANGLE_PAIR`] and the closer bills nothing (#1559). The grammars +/// give the brackets the same kind ids as the comparison operators, so +/// only the parent tells the two apart, and the opener is keyed by text +/// rather than by kind to keep it out of the comparison `<`'s +/// vocabulary entry. pub(crate) struct GenericAngleKinds { /// The nodes whose brackets enclose types: type-argument, /// type-parameter and template lists, each grammar's other spelling diff --git a/big-code-analysis-ast/src/getter/rust.rs b/big-code-analysis-ast/src/getter/rust.rs index 27f5ea6e..df7d21b4 100644 --- a/big-code-analysis-ast/src/getter/rust.rs +++ b/big-code-analysis-ast/src/getter/rust.rs @@ -6,10 +6,11 @@ use super::*; // `for_lifetimes` (`for<'a> fn(&'a u8)`) and `use_bounds` // (`impl Tr + use<'a, T>`) are generic-parameter lists under other // names, and a qualified path's `bracketed_type` (`::f`, -// `::f`) encloses a type (#1567). `LT2` is the `token(prec(1, '<'))` spelling `type_arguments` -// and `use_bounds` open with; the parser reports it as `LT`, so it is -// listed defensively, here and in the operator arm, and its absence is -// pinned by `rust_generic_opener_alias_never_reaches_kind_id`. +// `::f`) encloses a type (#1567). `LT2` is the +// `token(prec(1, '<'))` spelling `type_arguments` and `use_bounds` open +// with; the parser reports it as `LT`, so it is listed defensively, +// here and in the operator arm, and its absence is pinned by +// `rust_generic_opener_alias_never_reaches_kind_id`. const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { lists: &[ Rust::TypeArguments as u16, diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 78f7a9b2..2ed4d0e6 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -705,7 +705,8 @@ tokens involved the same way they spell real operators: no notion of a decision, so a `<` is the same vocabulary entry whether it is a comparison, a JSX tag delimiter, a Lua `` attribute, the declared name of a C# `operator <`, or a Perl `` - readline — and `bca ops` reports every one of them. This is the deliberate counterpart to + readline — and `bca ops` reports every one of them. This is the + deliberate counterpart to [the ABC deviation](#abc-per-language-deviations) that excludes the same tokens from `conditions`: ABC asks whether a branch is taken, and these are not branches; Halstead asks what alphabet the source uses, @@ -752,28 +753,31 @@ tokens involved the same way they spell real operators: literal's receiver. Groovy reserves these words, so as a member name or a label (`x.int`, `[int: 1]`) they stay operands. Java's `boolean` is listed as `boolean` rather than its node kind - `boolean_type`, so both languages report it alike. #1561 made TypeScript's type-level operator - keywords operators, as their value-level siblings `as`, `typeof` - and `extends` already were: `satisfies`, `keyof`, `infer`, and the - `is` and `asserts` of a type predicate. #1563 made TypeScript's - member modifiers operators, as C# and Kotlin bill theirs: `public`, - `private`, `protected`, `override`, `readonly`, `abstract`, - `accessor` and `static`, each the keyword and never its - `accessibility_modifier` or `override_modifier` wrapper. JavaScript - bills `static` too. The same words used as names (`let readonly = 1`, - `obj.static`) stay operands. #1564 made Kotlin's `typealias`, `init`, - `where`, `by`, `get` and `set` operators, as C# bills its `get`, - `set`, `init` and `where`. The `@get:` and `@set:` annotation targets - bill nothing, like `@field:`. The backing `field` in an accessor is an - operand, and the words used as names (`val by = 1`, `x.get(i)`) stay - operands. #1565 made Java and Groovy bill every modifier, where they - had billed only `abstract`, `final`, `transient` and `synchronized`: - the visibility keywords, `static`, `native`, `strictfp`, `volatile`, - `sealed` and `non-sealed`, plus the `permits` clause keyword beside - `extends` and `implements`. Java's are the keywords, never their - `modifiers` wrapper. The `static` of `import static` and `requires - static` bills nothing, as `import` does not, and the words used as - names (`var sealed = 1`) stay operands. + `boolean_type`, so both languages report it alike. #1561 made + TypeScript's type-level operator keywords operators, as their + value-level siblings `as`, `typeof` and `extends` already were: + `satisfies`, `keyof`, `infer`, and the `is` and `asserts` of a type + predicate. #1563 made TypeScript's member modifiers operators, as C# + and Kotlin bill theirs: `public`, `private`, `protected`, `override`, + `readonly`, `abstract`, `accessor` and `static`, each the keyword and + never its `accessibility_modifier` or `override_modifier` wrapper. + JavaScript and Mozjs bill `static` too. Their grammars lex + `static get` followed by a newline as one `static get` token, which + bills as one operator, so that spelling scores one entry where + `static get y()` on one line bills `static` and `get`. The same words used as names + (`let readonly = 1`, `obj.static`) stay operands. #1564 made Kotlin's + `typealias`, `init`, `where`, `by`, `get` and `set` operators, as C# + bills its `get`, `set`, `init` and `where`. The `@get:` and `@set:` + annotation targets bill nothing, like `@field:`. The backing `field` + in an accessor is an operand, and the words used as names + (`val by = 1`, `x.get(i)`) stay operands. #1565 made Java and Groovy bill every + modifier, where they had billed only `abstract`, `final`, `transient`, + `synchronized` and `default`: the visibility keywords, `static`, + `native`, `strictfp`, `volatile`, `sealed` and `non-sealed`, plus the + `permits` clause keyword beside `extends` and `implements`. Java's are + the keywords, never their `modifiers` wrapper. The `static` of + `import static` and `requires static` bills nothing, as `import` does not, and + the words used as names (`var sealed = 1`) stay operands. - **Angle brackets that enclose types are one bracket pair.** The `<` and `>` around type arguments and type parameters count as one `<>` operator, billed at the `<` as `()` is at the `(`, and the `>` counts @@ -783,8 +787,8 @@ tokens involved the same way they spell real operators: `template ` and `std::vector`; and the type-argument and type-parameter lists of Java, Groovy, C#, Kotlin, TypeScript and TSX, plus C#'s `delegate*` pointer type and Kotlin's - `super` (#1567). Objective-C has four generic lists: `NSArray`, - `id`, the protocol list in + `super` (#1567). Objective-C has four generic lists: + `NSArray`, `id`, the protocol list in `@interface A : NSObject `, and a type passed as a call argument, `f(NSArray)`. A comparison's `<` and `>` and a shift's `<<` and `>>` stay separate operators, and a C++ `>>` that closes two template lists @@ -798,9 +802,9 @@ tokens involved the same way they spell real operators: grammar parses as an assignment and which still bills `<>` and `=`. Go's generics use square brackets and already billed `[]`. Angle brackets around anything other than types still bill `<` and `>`: the JSX, Lua and Perl - constructs above. So does a Kotlin generic call with arguments, `id(a)`, - which tree-sitter-kotlin-ng parses as two comparisons (#1394), and a - generic inside a Rust macro call, `vec![Vec::::new()]`, whose + constructs above. So does a Kotlin generic call with arguments, + `id(a)`, which tree-sitter-kotlin-ng parses as two comparisons + (#1394), and a generic inside a Rust macro call, `vec![Vec::::new()]`, whose token tree tree-sitter-rust leaves unparsed; a `>>` closing two lists there bills as the shift. Before #1559 each bracket billed under the comparison's own entry, diff --git a/big-code-analysis-book/src/recipes/thresholds.md b/big-code-analysis-book/src/recipes/thresholds.md index 1c68f908..05dc6ae2 100644 --- a/big-code-analysis-book/src/recipes/thresholds.md +++ b/big-code-analysis-book/src/recipes/thresholds.md @@ -137,17 +137,19 @@ functions. Most languages need no override. The cases that do fall into three groups. **Procedural languages with large dispatch functions.** C, Tcl, Bash, Lua, Perl, and Go all run two -to three times the defaults. These languages lack exceptions or discourage them, so error handling -is inline branching, and they favour long `switch`-style dispatch over polymorphism. The default -`cognitive` limit flags 5% to 15% of their functions, against 3% in the median language. Raise -`cognitive` first. `halstead.effort` carries the rest of the excess for C, Go, and Lua; Tcl, Bash, -and Perl measure close to the default on it. +to three times the `cognitive` default and above the `cyclomatic` one. These languages lack +exceptions or discourage them, so error handling is inline branching, and they favour long +`switch`-style dispatch over polymorphism. The default `cognitive` limit flags 5% to 15% of their +functions, against 3% in the median language. Raise `cognitive` first. `halstead.effort` carries the +rest of the excess for C, Go, and Lua; Tcl, Bash, and Perl measure close to the default on it. In a single-language repository this is the whole `[thresholds]` table; in a mixed one it belongs under the language it describes, or it drags every other language up with it. ```toml -# Gating C. The same shape suits Tcl, Bash, Lua, Perl, and Go. +# Gating C. Tcl, Bash, Lua, Perl, and Go share its cognitive and +# cyclomatic shape, but not its effort: Go and Lua measure near 90000, +# and Tcl, Bash, and Perl near the default. [thresholds.lang.c] cognitive = 30 cyclomatic = 20 diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 775586af..b68d7be9 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -3754,17 +3754,16 @@ mod tests { ), ("argument list", "f(a, b);", "() , ;", "a b f"), ]; - let sorted = |words: &[String]| { - let mut words: Vec<&str> = words.iter().map(String::as_str).collect(); - words.sort_unstable(); - words.join(" ") - }; for (label, source, operators, operands) in ROWS { + // `bca ops` sorts its vocabulary, so the observation is + // compared as returned; the rows are written in that order. + let operators: Vec<&str> = operators.split(' ').collect(); + let operands: Vec<&str> = operands.split(' ').collect(); let ops = ops_of::(source, "foo.cs"); - assert_eq!(sorted(&ops.operators), *operators, "{label}: {source:?}"); - assert_eq!(sorted(&ops.operands), *operands, "{label}: {source:?}"); - let n1 = operators.split(' ').count() as u64; - let n2 = operands.split(' ').count() as u64; + assert_eq!(ops.operators, operators, "{label}: {source:?}"); + assert_eq!(ops.operands, operands, "{label}: {source:?}"); + let n1 = operators.len() as u64; + let n2 = operands.len() as u64; assert_halstead_counts::(source, "foo.cs", [n1, n1, n2, n2], label); } } @@ -6716,9 +6715,9 @@ end", /// #1565: Groovy bills every modifier, as Java now does. Its grammar /// has no `modifiers` wrapper, so each leaf sits directly under its - /// declaration. `void` and `int` are type names and bill as - /// operands, so they are absent from these operator counts. The last - /// two rows are controls. + /// declaration. The rows declare with `def`, so the only primitive + /// type counted is `void`, an operator since #1568. The last two + /// rows are controls. #[cfg(feature = "groovy")] #[test] fn groovy_modifiers_are_operators_1565() { @@ -6824,6 +6823,27 @@ end", ); } + /// #1565's oracle: Groovy accepts Java's modifier-heavy fixture + /// verbatim, and the two languages bill it alike, vocabulary and + /// multiplicities. Neither side's expectation is typed by the fix + /// alone: a modifier one getter drops shows as a disagreement. + #[cfg(all(feature = "groovy", feature = "java"))] + #[test] + fn groovy_modifiers_match_java_1565() { + let source = "public abstract class C { private static final int a = 1; \ + protected transient int b; public synchronized void m() {} }\n"; + let groovy = ops_of::(source, "foo.groovy"); + let java = ops_of::(source, "Foo.java"); + assert_eq!(groovy.operators, java.operators); + assert_eq!(groovy.operands, java.operands); + // `public` x2, `abstract`, `class`, `{}` x2, `private`, `static`, + // `final`, `int` x2, `=`, `;` x2, `protected`, `transient`, + // `synchronized`, `void`, `()` + let rows = [("modifier fixture", source, [15, 19])]; + assert_operator_rows::("foo.groovy", &rows); + assert_operator_rows::("Foo.java", &rows); + } + /// #1552 rows shared by the two C++ clones. The template-parameter /// rows pin the synonym: `class T` is `typename T`, and the two /// spellings score alike. Both were unbilled until #1557 billed From bacb45623f01a77098772030a59d6517a2170bf3 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:59:38 -0700 Subject: [PATCH 11/34] fix(getter/cpp): bill C-family declaration modifiers C, C++ and Mozcpp billed no storage class, function specifier or type qualifier, while every other class language bills its modifiers. Each keyword leaf is now an operator: static, extern, register, inline, thread_local, const, volatile, restrict, _Atomic, constexpr, alignas, _Noreturn and their vendor spellings (__inline, __restrict__, __thread, ...), C's storage class auto, and C++'s virtual, mutable, constinit, consteval, explicit, friend, the access labels, override, final and the noexcept of an exception specification. The storage_class_specifier, type_qualifier, virtual_specifier, access_specifier, explicit_function_specifier and noexcept wrappers stay unlisted, so each modifier bills once; a member-function trailer const and a pointer's const each bill where they appear. None of the words has a numeric-suffix alias. override and final used as names parse as identifiers and stay operands. C++'s auto is a placeholder type and stays unbilled; attributes, MS calling conventions and the Gecko MOZ_* macros are unchanged. The DeepSpeech corpus moves in 884 snapshot files, Halstead operator counts and MI only: operands and structure are unchanged. Fixes #1578 --- big-code-analysis-ast/src/getter/c.rs | 20 +- big-code-analysis-ast/src/getter/cpp.rs | 28 ++ big-code-analysis-ast/src/getter/mozcpp.rs | 8 + big-code-analysis-book/src/metrics.md | 16 +- src/metrics/halstead.rs | 277 +++++++++++++++++- ...put_formats__csv_test__csv_cpp_widget.snap | 8 +- tests/parity/cpp_mozcpp_parity.rs | 41 +++ tests/repositories/big-code-analysis-output | 2 +- 8 files changed, 389 insertions(+), 11 deletions(-) diff --git a/big-code-analysis-ast/src/getter/c.rs b/big-code-analysis-ast/src/getter/c.rs index 8fb6520b..2a2009a3 100644 --- a/big-code-analysis-ast/src/getter/c.rs +++ b/big-code-analysis-ast/src/getter/c.rs @@ -86,7 +86,25 @@ impl Getter for CCode { // C++-only forms, so leaving them out here would score a // header differently by the grammar that parsed it. The // derivation is on the Cpp arm. - | Struct | Union | Enum | Typedef => TokenRole::Operator, + | Struct | Union | Enum | Typedef + // Storage classes, function specifiers and type qualifiers, + // billed as every class language bills its modifiers + // (#1578). Each is the keyword leaf of an unclassified + // `storage_class_specifier` / `type_qualifier` / + // `alignas_qualifier`, so it bills once, as `const` does in + // a pointer declarator (`int * const p`). The vendor + // spellings (`__inline`, `__restrict__`, `__thread`, …) are + // kinds of their own and bill under their own name. `auto` + // is a storage class in C, the only place this grammar + // emits it. `__extension__` bills in an expression too, + // where it is the prefix of an `extension_expression`. + // Attributes (`__attribute__`, `__declspec`, `[[…]]`) and + // the MS calling conventions stay unbilled: they annotate a + // declaration rather than spell part of it. + | Static | Extern | Register | Auto | Inline | Inline2 | Inline3 | Forceinline + | ThreadLocal | Thread | Const | Constexpr | Volatile | Restrict | Restrict2 + | Atomic | Noreturn | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension + => TokenRole::Operator, // `CharLiteral` joins the operand list here for the whole // C family — `cpp.rs`, `mozcpp.rs` and `objc.rs` carry the // same arm and point back at this note (#1316). Before it a diff --git a/big-code-analysis-ast/src/getter/cpp.rs b/big-code-analysis-ast/src/getter/cpp.rs index 89ac98ee..7c17ee15 100644 --- a/big-code-analysis-ast/src/getter/cpp.rs +++ b/big-code-analysis-ast/src/getter/cpp.rs @@ -203,6 +203,34 @@ impl Getter for CppCode { // spellings of `template ` still score alike, now // both billed. | Class | Struct | Union | Enum | Namespace | Template | Typename | Typedef | Using + // C's storage classes, function specifiers and type + // qualifiers (#1578; the derivation is on the C arm), plus + // C++'s own modifiers: `virtual`, `mutable`, `constinit`, + // `consteval`, `explicit`, `friend`, the access specifiers + // and the `override` / `final` virt-specifiers, and the + // `noexcept` of an exception specification. Each is the + // keyword leaf of an unclassified wrapper + // (`storage_class_specifier`, `type_qualifier`, + // `virtual_specifier`, `access_specifier`, + // `explicit_function_specifier`, the `noexcept` node that + // `Noexcept` names and `Noexcept2` is the leaf of) or sits + // bare in its declaration (`virtual`, `friend`), so each + // bills once. An access label bills as Java's `public` does, + // beside the `:` that already billed. `const` bills in a + // member-function trailer and a pointer declarator alike, + // and `constexpr` in `if constexpr`, which it qualifies. + // + // `override` and `final` are contextual: used as a name + // (`int override = 1;`) they parse as an `identifier` and + // stay operands, so they need no guard. `auto` is not here: + // in C++ it is a placeholder type, not a storage class, and + // the grammar emits it only under + // `placeholder_type_specifier`. + | Static | Extern | Register | Inline | Inline2 | Inline3 | Forceinline + | ThreadLocal | Thread | Const | Constexpr | Volatile | Restrict | Restrict2 + | Atomic | Noreturn | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension + | Virtual | Mutable | Constinit | Consteval | Explicit | Friend | Public | Private + | Protected | Override | Final | Noexcept2 => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only diff --git a/big-code-analysis-ast/src/getter/mozcpp.rs b/big-code-analysis-ast/src/getter/mozcpp.rs index c822ec32..c16140e2 100644 --- a/big-code-analysis-ast/src/getter/mozcpp.rs +++ b/big-code-analysis-ast/src/getter/mozcpp.rs @@ -136,6 +136,14 @@ impl Getter for MozcppCode { // The type-declaration keywords — the derivation is on the // Cpp twin (#1552, #1557). | Class | Struct | Union | Enum | Namespace | Template | Typename | Typedef | Using + // The declaration modifiers — the derivation is on the Cpp + // twin (#1578). The Gecko `MOZ_*` annotation macros stay + // unbilled, as attributes do. + | Static | Extern | Register | Inline | Inline2 | Inline3 | Forceinline + | ThreadLocal | Thread | Const | Constexpr | Volatile | Restrict | Restrict2 + | Atomic | Noreturn | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension + | Virtual | Mutable | Constinit | Consteval | Explicit | Friend | Public | Private + | Protected | Override | Final | Noexcept2 => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 2ed4d0e6..89f4cec4 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -777,7 +777,21 @@ tokens involved the same way they spell real operators: `permits` clause keyword beside `extends` and `implements`. Java's are the keywords, never their `modifiers` wrapper. The `static` of `import static` and `requires static` bills nothing, as `import` does not, and - the words used as names (`var sealed = 1`) stay operands. + the words used as names (`var sealed = 1`) stay operands. #1578 made + C, C++ and Mozcpp bill every storage class, function specifier and + type qualifier: `static`, `extern`, `register`, `inline`, + `thread_local`, `const`, `volatile`, `restrict`, `_Atomic`, + `constexpr`, `alignas` and their vendor spellings (`__inline`, + `__restrict__`, `__thread`), plus C++'s `virtual`, `mutable`, + `constinit`, `consteval`, `explicit`, `friend`, the access labels, + `override`, `final` and `noexcept`. Each is the keyword, never its + `storage_class_specifier`, `type_qualifier` or `virtual_specifier` + wrapper, so the `const` of `void f() const` and of `int * const p` + each bill once. C bills its storage class `auto`. C++'s `auto` is a + placeholder type and stays unbilled. `extern "C"` bills `extern` and + keeps `"C"` an operand. Attributes, the MS calling conventions and + the Gecko `MOZ_*` macros bill nothing, and `override` and `final` + used as names (`int override = 1;`) stay operands. - **Angle brackets that enclose types are one bracket pair.** The `<` and `>` around type arguments and type parameters count as one `<>` operator, billed at the `<` as `()` is at the `(`, and the `>` counts diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index b68d7be9..74239250 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -602,10 +602,12 @@ mod tests { /// among the operands would pass on that. The operator side is what /// pins the removal. #[cfg(any( + feature = "cpp", feature = "csharp", feature = "groovy", feature = "java", feature = "kotlin", + feature = "mozcpp", feature = "typescript" ))] #[track_caller] @@ -6917,6 +6919,273 @@ end", assert_operator_rows::("foo.c", ROWS); } + /// #1578: C bills each storage class, function specifier and type + /// qualifier as its keyword leaf, never as the + /// `storage_class_specifier` / `type_qualifier` wrapper. Before the + /// fix none of them billed. + #[cfg(feature = "c")] + #[test] + fn c_modifiers_are_operators_1578() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `static`, `extern`, `register`, `auto`, `int` x4, `;` x4 + ( + "storage classes", + "static int a;\nextern int b;\nregister int c;\nauto int d;\n", + [6, 12], + ), + // `const`, `volatile`, `int` x3, `;` x3, `*`, `restrict`, + // `_Atomic` + ( + "qualifiers", + "const volatile int a;\nint * restrict p;\n_Atomic int g;\n", + [7, 11], + ), + // `inline`, `int`, `()` x2, `void` x3, `;` x2, `_Noreturn` + ( + "function specifiers", + "inline int f(void);\n_Noreturn void g(void);\n", + [6, 10], + ), + // `__inline`, `int` x3, `()`, `void`, `;` x3, `__thread`, `*`, + // `__restrict__`: each vendor spelling is its own operator + ( + "vendor spellings", + "__inline int f(void);\n__thread int t;\nint * __restrict__ q;\n", + [8, 12], + ), + // `int`, `*`, `const`, `=`, `;`: a pointer's own qualifier + ("pointer qualifier", "int * const p = 0;\n", [5, 5]), + // `_Alignas`, `()`, `int`, `;` + ("alignas", "_Alignas(8) int m;\n", [4, 4]), + // `int`, `=`, `__extension__`, `;`: the expression prefix + ("extension expression", "int q = __extension__ 1;\n", [4, 4]), + ]; + assert_operator_rows::("foo.c", ROWS); + } + + /// #1578: the issue's C fixture, by vocabulary. + #[cfg(feature = "c")] + #[test] + fn c_modifier_vocabulary_1578() { + let ops = ops_of::( + "static inline const volatile int a = 1;\nextern int b;\nregister int c;\n", + "foo.c", + ); + assert_eq!( + ops.operators, + [ + ";", "=", "const", "extern", "inline", "int", "register", "static", "volatile", + ] + ); + assert_eq!(ops.operands, ["1", "a", "b", "c"]); + } + + /// #1578 rows shared by the two C++ clones: C's modifiers plus C++'s + /// own. Each bills as its keyword leaf, never as the + /// `access_specifier` / `virtual_specifier` / `type_qualifier` / + /// `explicit_function_specifier` wrapper. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const CPP_MODIFIER_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}`, `public`, `protected`, `private`, `:` x3, + // `int` x3, `;` x4 + ( + "access labels", + "class C { public: int a; protected: int b; private: int c; };\n", + [8, 15], + ), + // `struct`, `{}`, `static`, `int` x2, `;` x4, `mutable`, + // `virtual`, `void`, `()`, `const`, `override`, `final` + ( + "member modifiers", + "struct S { static int a; mutable int b; virtual void f() const override final; };\n", + [12, 16], + ), + // `struct`, `{}`, `friend`, `class`, `;` x3, `explicit`, `()`, + // `int` + ( + "friend and explicit", + "struct S { friend class D; explicit S(int); };\n", + [8, 10], + ), + // `extern`, `inline`, `constexpr`, `int`, `()`, `{}`, `return`, + // `;`: the `"C"` is an operand + ( + "linkage specification", + "extern \"C\" inline constexpr int g() { return 1; }\n", + [8, 8], + ), + // `consteval`, `int` x2, `()`, `;` x2, `constinit`, + // `thread_local`, `=` + ( + "C++20 specifiers", + "consteval int h();\nconstinit thread_local int t = 1;\n", + [7, 9], + ), + // `void` x2, `()` x3, `noexcept` x2, `;` x2: the leaf bills, the + // `noexcept` wrapper does not + ( + "exception specification", + "void k() noexcept;\nvoid k2() noexcept(true);\n", + [4, 9], + ), + // `struct`, `final`, `{}` x3, `;` x3, `class` x2, `:` x2, + // `public`, `private`, `virtual` + ( + "base clauses", + "struct B final {};\nclass E : public B {};\nclass F : private virtual B {};\n", + [9, 15], + ), + // `void`, `()` x2, `{}` x2, `if`, `constexpr` + ( + "if constexpr", + "void f() { if constexpr (true) {} }\n", + [5, 7], + ), + // `inline`, `namespace`, `{}` + ("inline namespace", "inline namespace v1 {}\n", [3, 3]), + // `int` x2, `=` x2, `;` x2: the contextual words as names add + // no operator + ( + "contextual names", + "int override = 1;\nint final = override;\n", + [3, 6], + ), + ]; + + /// One call per clone, so reverting either arm fails its own row; + /// Mozcpp owns no file extension, so this is its only lib coverage. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_modifiers_are_operators_1578() { + assert_operator_rows::("foo.cpp", CPP_MODIFIER_ROWS); + assert_operator_rows::("foo.cpp", CPP_MODIFIER_ROWS); + } + + /// #1578: the issue's C++ fixture, by vocabulary, in both clones, + /// plus `override` and `final` used as names. No getter edit makes + /// that second half fail, since the arms carry no guard on those + /// words: it pins the grammar emitting an `identifier` for a name, + /// so a bump that routes one to the keyword kind trips it. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_modifier_vocabulary_1578() { + const SOURCE: &str = "class C { public: static int a; \ + virtual void f() const override final; mutable int b; };\n\ + extern \"C\" inline constexpr int g() { return 1; }\n"; + const NAMES: &str = "int override = 1;\nint final() { return override; }\n"; + let expected_operators = [ + "()", + ":", + ";", + "class", + "const", + "constexpr", + "extern", + "final", + "inline", + "int", + "mutable", + "override", + "public", + "return", + "static", + "virtual", + "void", + "{}", + ]; + let expected_operands = ["\"C\"", "1", "C", "a", "b", "f", "g"]; + + let cpp = ops_of::(SOURCE, "foo.cpp"); + assert_eq!(cpp.operators, expected_operators); + assert_eq!(cpp.operands, expected_operands); + let mozcpp = ops_of::(SOURCE, "foo.cpp"); + assert_eq!(mozcpp.operators, expected_operators); + assert_eq!(mozcpp.operands, expected_operands); + + assert_keywords_are_operands_only::(NAMES, "foo.cpp", &["override", "final"]); + assert_keywords_are_operands_only::(NAMES, "foo.cpp", &["override", "final"]); + } + + /// #1578: every modifier spelling the arms list bills under its own + /// name, one fixture line apiece, so dropping any one kind from an + /// arm fails its own entry. The row tests above pin the counts; this + /// one pins the spellings they leave out. + #[cfg(any(feature = "c", feature = "cpp", feature = "mozcpp"))] + #[track_caller] + fn assert_modifier_spellings_bill(file: &str, rows: &[(&str, &str)]) { + for (spelling, source) in rows { + let ops = ops_of::(source, file); + assert!( + ops.operators.iter().any(|o| o == spelling), + "{file}: `{spelling}` must be an operator in {source:?}; operators were {:?}", + ops.operators + ); + assert!( + !ops.operands.iter().any(|o| o == spelling), + "{file}: `{spelling}` must not be an operand; operands were {:?}", + ops.operands + ); + } + } + + /// The spellings C and C++ share, each in a declaration both + /// grammars parse without error. + #[cfg(any(feature = "c", feature = "cpp", feature = "mozcpp"))] + const C_FAMILY_MODIFIER_SPELLINGS: &[(&str, &str)] = &[ + ("static", "static int a;\n"), + ("extern", "extern int a;\n"), + ("register", "register int a;\n"), + ("inline", "inline int f(void);\n"), + ("__inline", "__inline int f(void);\n"), + ("__inline__", "__inline__ int f(void);\n"), + ("__forceinline", "__forceinline int f(void);\n"), + ("thread_local", "thread_local int a;\n"), + ("__thread", "__thread int a;\n"), + ("const", "const int a = 1;\n"), + ("constexpr", "constexpr int a = 1;\n"), + ("volatile", "volatile int a;\n"), + ("restrict", "int * restrict p;\n"), + ("__restrict__", "int * __restrict__ p;\n"), + ("_Atomic", "_Atomic int a;\n"), + ("_Noreturn", "_Noreturn void f(void);\n"), + ("noreturn", "noreturn void f(void);\n"), + ("_Nonnull", "int * _Nonnull p;\n"), + ("alignas", "alignas(8) int a;\n"), + ("_Alignas", "_Alignas(8) int a;\n"), + ("__extension__", "__extension__ int a;\n"), + ]; + + #[cfg(feature = "c")] + #[test] + fn c_modifier_spellings_bill_1578() { + assert_modifier_spellings_bill::("foo.c", C_FAMILY_MODIFIER_SPELLINGS); + // C's storage class `auto`; C++ has none (see the Cpp arm). + assert_modifier_spellings_bill::("foo.c", &[("auto", "auto int a;\n")]); + } + + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_modifier_spellings_bill_1578() { + const CPP_ONLY: &[(&str, &str)] = &[ + ("virtual", "struct S { virtual void f(); };\n"), + ("mutable", "struct S { mutable int a; };\n"), + ("constinit", "constinit int a = 1;\n"), + ("consteval", "consteval int f() { return 1; }\n"), + ("explicit", "struct S { explicit S(int); };\n"), + ("friend", "struct S { friend class D; };\n"), + ("public", "class C { public: int a; };\n"), + ("private", "class C { private: int a; };\n"), + ("protected", "class C { protected: int a; };\n"), + ("override", "struct S { void f() override; };\n"), + ("final", "struct S final {};\n"), + ("noexcept", "void f() noexcept;\n"), + ]; + for rows in [C_FAMILY_MODIFIER_SPELLINGS, CPP_ONLY] { + assert_modifier_spellings_bill::("foo.cpp", rows); + assert_modifier_spellings_bill::("foo.cpp", rows); + } + } + /// #1552: ObjC bills C's `struct` and its own class and protocol /// declaration keywords. `@class` already billed its `@` and stays /// as it was. @@ -11895,13 +12164,13 @@ S s; int a = s[1]; auto t = co_await s; #[cfg(any(feature = "cpp", feature = "mozcpp"))] #[track_caller] fn assert_cpp_operator_names(label: &str) { - // Operators (n1 = 17, N1 = 40): `operator` 5 (four + // Operators (n1 = 18, N1 = 41): `operator` 5 (four // `operator_name`s and the `operator_cast`); the names `[]` 1, // `()` 1, `""` 1; `co_await` 2 (the name and the expression); // the `(` of a parameter list 5; `;` 9; `int` 3; `long` 3; // `bool` 2; `=` 2; `{` 1; `void` 1; `unsigned` 1; `::` 1; the - // subscript `[` 1; `struct` 1 (#1552). `auto` and `const` are in - // no arm. + // subscript `[` 1; `struct` 1 (#1552); the trailing `const` 1 + // (#1578). `auto` is in no arm. // // Operands (n2 = 6, N2 = 9): `s` 3, `S` 2 (the qualifier in // `S::operator bool` is a namespace identifier, which #1096 @@ -11909,7 +12178,7 @@ S s; int a = s[1]; auto t = co_await s; // // Before #1296: (12, 30, 6, 9) with `struct`, the five operator // kinds billed nowhere. - assert_halstead_counts::(CPP_OPERATOR_NAMES, "ops.cpp", [17, 40, 6, 9], label); + assert_halstead_counts::(CPP_OPERATOR_NAMES, "ops.cpp", [18, 41, 6, 9], label); // `operator[]` / `operator()` are kinds of their own, so they sit // beside the subscript `[` and the parameter-list `(` as two more diff --git a/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap b/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap index ca5892f2..10b604cd 100644 --- a/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap +++ b/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap @@ -3,8 +3,8 @@ source: tests/output_formats/csv_test.rs expression: out --- path,space_name,space_kind,start_line,end_line,cognitive.sum,cognitive.average,cognitive.min,cognitive.max,cyclomatic.sum,cyclomatic.average,cyclomatic.min,cyclomatic.max,cyclomatic.modified.sum,cyclomatic.modified.average,cyclomatic.modified.min,cyclomatic.modified.max,halstead.unique_operators,halstead.total_operators,halstead.unique_operands,halstead.total_operands,halstead.length,halstead.estimated_program_length,halstead.purity_ratio,halstead.vocabulary,halstead.volume,halstead.difficulty,halstead.level,halstead.effort,halstead.time,halstead.bugs,loc.sloc,loc.ploc,loc.lloc,loc.cloc,loc.blank,loc.sloc_average,loc.ploc_average,loc.lloc_average,loc.cloc_average,loc.blank_average,loc.sloc_min,loc.sloc_max,loc.cloc_min,loc.cloc_max,loc.ploc_min,loc.ploc_max,loc.lloc_min,loc.lloc_max,loc.blank_min,loc.blank_max,nom.functions,nom.closures,nom.functions_average,nom.closures_average,nom.total,nom.average,nom.functions_min,nom.functions_max,nom.closures_min,nom.closures_max,nargs.function_args,nargs.closure_args,nargs.function_args_average,nargs.closure_args_average,nargs.total,nargs.average,nargs.function_args_min,nargs.function_args_max,nargs.closure_args_min,nargs.closure_args_max,nexits.sum,nexits.average,nexits.min,nexits.max,tokens.sum,tokens.average,tokens.min,tokens.max,abc.assignments,abc.branches,abc.conditions,abc.magnitude,abc.assignments_average,abc.branches_average,abc.conditions_average,abc.assignments_min,abc.assignments_max,abc.branches_min,abc.branches_max,abc.conditions_min,abc.conditions_max,wmc.class_wmc_sum,wmc.interface_wmc_sum,wmc.total,npm.class_npm_sum,npm.interface_npm_sum,npm.class_methods,npm.interface_methods,npm.class_coa,npm.interface_coa,npm.total,npm.total_methods,npm.coa,npa.class_npa_sum,npa.interface_npa_sum,npa.class_attributes,npa.interface_attributes,npa.class_cda,npa.interface_cda,npa.total,npa.total_attributes,npa.cda,mi.original,mi.sei,mi.visual_studio -widget.cc,widget.cc,unit,1,10,0,0,0,0,5,2.5,1,1,5,2.5,1,1,10,20,6,9,29,48.72905595320056,1.680312274248295,16,116,7.5,0.13333333333333333,870,48.333333333333336,0.030377934896425568,10,9,2,0,1,2,1.8,0.4,0,0.2,1,10,0,0,1,9,1,2,0,1,2,0,0.4,0,2,0.4,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,7.6,0,12,1,0,0,1,0.2,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,107.82945249974337,80.37326368816134,63.05815935657507 -widget.cc,ns,namespace,2,10,0,0,0,0,4,2,1,1,4,2,1,1,10,20,6,9,29,48.72905595320056,1.680312274248295,16,116,7.5,0.13333333333333333,870,48.333333333333336,0.030377934896425568,9,9,2,0,0,2.25,2.25,0.5,0,0,1,9,0,0,1,9,1,2,0,0,2,0,0.5,0,2,0.5,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,9.5,5,12,1,0,0,1,0.25,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,109.76629285340017,83.06571380197117,64.19081453415215 -widget.cc,Widget,class,3,9,0,0,0,0,3,1.5,1,1,3,1.5,1,1,9,17,5,8,25,40.13896548741762,1.6055586194967049,14,95.1838730514401,7.2,0.1388888888888889,685.3238859703688,38.07354922057604,0.02591051177740593,7,7,2,0,0,2.3333333333333335,2.3333333333333335,0.6666666666666666,0,0,1,7,0,0,1,7,1,2,0,0,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,33,11,10,12,1,0,0,1,0.3333333333333333,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,115.09604084632235,90.65309494114345,67.3076262259195 -widget.cc,value,function,5,5,0,0,0,0,1,1,1,1,1,1,1,1,5,5,2,2,7,13.60964047443681,1.944234353490973,7,19.651484454403228,2.5,0.4,49.12871113600807,2.7293728408893374,0.004471319041702271,1,1,1,0,0,1,1,1,0,0,1,1,0,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,10,10,10,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,155.2836050228807,148.4278547652622,90.80912574437467 +widget.cc,widget.cc,unit,1,10,0,0,0,0,5,2.5,1,1,5,2.5,1,1,13,23,6,9,32,63.61549134016113,1.9879841043800353,19,135.93368043019473,9.75,0.10256410256410256,1325.3533841943986,73.63074356635548,0.040219179477315764,10,9,2,0,1,2,1.8,0.4,0,0.2,1,10,0,0,1,9,1,2,0,1,2,0,0.4,0,2,0.4,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,7.6,0,12,1,0,0,1,0.2,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,107.0048524543984,79.18361729202537,62.57593710783532 +widget.cc,ns,namespace,2,10,0,0,0,0,4,2,1,1,4,2,1,1,13,23,6,9,32,63.61549134016113,1.9879841043800353,19,135.93368043019473,9.75,0.10256410256410256,1325.3533841943986,73.63074356635548,0.040219179477315764,9,9,2,0,0,2.25,2.25,0.5,0,0,1,9,0,0,1,9,1,2,0,0,2,0,0.5,0,2,0.5,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,9.5,5,12,1,0,0,1,0.25,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,108.94169280805521,81.8760674058352,63.70859228541241 +widget.cc,Widget,class,3,9,0,0,0,0,3,1.5,1,1,3,1.5,1,1,12,20,5,8,28,54.62919048309068,1.9510425172532386,17,114.4489595550095,9.6,0.10416666666666667,1098.710011728091,61.03944509600506,0.03549229914069982,7,7,2,0,0,2.3333333333333335,2.3333333333333335,0.6666666666666666,0,0,1,7,0,0,1,7,1,2,0,0,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,33,11,10,12,1,0,0,1,0.3333333333333333,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,114.13758501653936,89.27033546860443,66.74712574066629 +widget.cc,value,function,5,5,0,0,0,0,1,1,1,1,1,1,1,1,6,6,2,2,8,17.509775004326936,2.188721875540867,8,24,3,0.3333333333333333,72,4,0.0057689982812296325,1,1,1,0,0,1,1,1,0,0,1,1,0,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,10,10,10,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,154.24412008219068,146.92819499625,90.20123981414659 widget.cc,set,function,6,6,0,0,0,0,1,1,1,1,1,1,1,1,6,6,3,4,10,20.264662506490403,2.0264662506490403,9,31.69925001442312,4,0.25,126.79700005769249,7.044277780982916,0.008413009854528818,1,1,1,0,0,1,1,1,0,0,1,1,0,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,12,12,12,12,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,152.79727628688778,144.8408406278255,89.35513233151332 diff --git a/tests/parity/cpp_mozcpp_parity.rs b/tests/parity/cpp_mozcpp_parity.rs index 73c9f03c..b03cb66c 100644 --- a/tests/parity/cpp_mozcpp_parity.rs +++ b/tests/parity/cpp_mozcpp_parity.rs @@ -200,3 +200,44 @@ fn cpp_and_mozcpp_agree_on_this() { "`this` must be billed as an operand: {cpp:?}" ); } + +#[cfg(all(feature = "cpp", feature = "mozcpp"))] +#[test] +fn cpp_and_mozcpp_agree_on_modifiers() { + // #1578 added the declaration modifiers to the operator arm of both + // `CppCode::get_op_type` and its `MozcppCode` clone. Mozcpp owns no + // file extension, so nothing else at the metric level exercises its + // copy. + // + // The fixture carries one of each wrapper the grammar puts a + // modifier in — `access_specifier`, `storage_class_specifier`, + // `virtual_specifier`, `type_qualifier` — beside a linkage + // specification whose `"C"` must stay an operand. + let source = "class C { public: static int a; virtual void f() const override final; \ + mutable int b; };\n\ + extern \"C\" inline constexpr int g() { return 1; }\n"; + + let cpp = metric_sums(LANG::Cpp, source, "cpp"); + let mozcpp = metric_sums(LANG::Mozcpp, source, "cpp"); + assert_eq!( + cpp, mozcpp, + "Cpp and Mozcpp must agree on modifier classification" + ); + + let get = |key: &str| metric(&cpp, key); + // Operators: `class`, `{` x2, `public`, `:`, `static`, `int` x3, + // `;` x5, `virtual`, `void`, `(` x2, `const`, `override`, `final`, + // `mutable`, `extern`, `inline`, `constexpr`, `return` -> N1 = 26. + // Before #1578 the ten modifiers billed nothing and this was 16. + assert_eq!( + get("halstead.operators"), + 26, + "each modifier must bill once as an operator: {cpp:?}" + ); + // Operands: `C`, `a`, `f`, `b`, `"C"`, `g`, `1` -> N2 = 7. + assert_eq!( + get("halstead.operands"), + 7, + "no modifier may bill as an operand: {cpp:?}" + ); +} diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index b415bc3b..3c754c5a 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit b415bc3bae8888a032ec99f78f3f2f7529fd0ead +Subproject commit 3c754c5af769cd3e13303e2b1094250736612f5b From 34fd71c1a73a8ab47b90ecc99a8e9e67ba7fc982 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:16:47 -0700 Subject: [PATCH 12/34] fix(getter/objc): bill C and ObjC declaration modifiers Objective-C billed none of C's storage classes, function specifiers or type qualifiers, so a .m scored a C declaration below a .c or .mm. The ObjC getter now bills C's list exactly as c.rs does (#1578), plus ObjC's own modifiers as keyword leaves: the @public / @private / @protected / @package ivar labels, @optional / @required, the ARC ownership qualifiers and __bridge casts, the nullability qualifiers, __kindof, __covariant / __contravariant, _Complex, and the in / out / inout / bycopy / byref / oneway method-type qualifiers. A property attribute has no keyword kind: it is the first identifier of a property_attribute, which now bills as an operator keyed by its text; a getter= / setter= value stays an operand. The nullable qualifier is text-keyed too, so it and the nullable attribute share one n1 entry. Apple SDK macros (NS_INLINE, FOUNDATION_EXPORT, IBOutlet, __unused, ...) stay unbilled, as Mozcpp's MOZ_* do. No corpus file is Objective-C, so no snapshot moves. Fixes #1579 --- big-code-analysis-ast/src/checker/objc.rs | 11 +- big-code-analysis-ast/src/getter/objc.rs | 60 ++++++- big-code-analysis-book/src/metrics.md | 13 +- src/metrics/halstead.rs | 202 +++++++++++++++++++++- 4 files changed, 279 insertions(+), 7 deletions(-) diff --git a/big-code-analysis-ast/src/checker/objc.rs b/big-code-analysis-ast/src/checker/objc.rs index 7f28559d..f26c7eeb 100644 --- a/big-code-analysis-ast/src/checker/objc.rs +++ b/big-code-analysis-ast/src/checker/objc.rs @@ -113,8 +113,17 @@ impl Checker for ObjcCode { impl_is_else_if_parent_clause!(Objc, IfStatement, ElseClause); + // Besides the primitive types, an `identifier` the getter bills is a + // property attribute word (`nonatomic`, `copy`), whose kind is no + // keyword, so keying it by text lists each word. The `nullable` + // type qualifier is keyed by text too, so it and the `nullable` + // property attribute share one `n1` entry rather than one per map + // (#1579). #[inline] fn is_primitive(node: &Node) -> bool { - node.kind_id() == Objc::PrimitiveType + matches!( + node.kind_id().into(), + Objc::PrimitiveType | Objc::Identifier | Objc::Nullable + ) } } diff --git a/big-code-analysis-ast/src/getter/objc.rs b/big-code-analysis-ast/src/getter/objc.rs index c94315d4..2d398f5b 100644 --- a/big-code-analysis-ast/src/getter/objc.rs +++ b/big-code-analysis-ast/src/getter/objc.rs @@ -40,6 +40,19 @@ fn objc_argument_list_misparse(list: &Node, _ancestors: Ancestors) -> bool { list.kind_id() == Objc::ArgumentList2 as u16 && list.has_error() } +// Whether `node` is the attribute word of `@property (nonatomic, copy, +// getter=isOn)`. The grammar has no keyword for any attribute: each is +// an `identifier`, the first child of its `property_attribute`, so the +// word is told from a name by position alone. The `isOn` of +// `getter=isOn` is a later child and stays an operand, the selector it +// names. `ObjcCode::is_primitive` keys the word by its text (#1579). +fn is_property_attribute_name(node: &Node, ancestors: Ancestors) -> bool { + ancestors.parent(node).is_some_and(|attribute| { + attribute.kind_id() == Objc::PropertyAttribute as u16 + && attribute.start_byte() == node.start_byte() + }) +} + impl Getter for ObjcCode { fn get_func_space_name<'a, 'tree>( node: &Node<'tree>, @@ -144,6 +157,40 @@ impl Getter for ObjcCode { // arm: it parses as an `@` token, already billed here, and a // separate `class` leaf, so the directive already bills one // operator. + // + // From `Static` onwards come the declaration modifiers + // (#1579), billed as every class language bills its + // modifiers: first C's storage classes, function specifiers + // and type qualifiers, the same list as `c.rs` (#1578), so a + // `.m` scores a C declaration as a `.c` does; then ObjC's + // own. Those are the ARC ownership qualifiers (`__weak`, + // `__strong`, `__block`, …) and `__bridge` casts, the + // nullability qualifiers in all three spellings + // (`nullable`, `_Nullable`, `__nullable`), `__kindof`, the + // `__covariant` / `__contravariant` variance of a generic + // parameter, the `_Complex` type keyword, which this grammar + // files as a qualifier, the `in` / `out` / `inout` / + // `bycopy` / `byref` / `oneway` method-type qualifiers (`in` + // already billed above), the `@public` / `@private` / + // `@protected` / `@package` ivar visibility labels and the + // `@optional` / `@required` protocol section labels. Each is + // the keyword leaf of an unclassified `storage_class_specifier` + // / `type_qualifier` / `protocol_qualifier` / + // `visibility_specification` or sits bare in its parent, so + // it bills once. None has a numeric-suffix alias. + // + // Left unbilled: the Apple SDK macros this grammar also lists + // as storage classes and qualifiers (`NS_INLINE`, + // `FOUNDATION_EXPORT`, `IBOutlet`, `__unused`, + // `__ptrauth_objc_*`, …), as Mozcpp's `MOZ_*` macros are, + // since each annotates a declaration and a `.c` file parses + // the same spelling as a name; and GNU's `__real` / + // `__imag`, complex-part operators the grammar misfiles as + // qualifiers. A variable named `nullable` parses as the + // qualifier before a MISSING name and bills it, as C's + // `int noreturn = 1;` bills `noreturn`: the grammar decides + // that, not this arm. A property attribute has no keyword + // kind at all; see `is_property_attribute_name`. DOT | LPAREN | LPAREN2 | COMMA | STAR | GTGT | COLON | SEMI | Return | Break | Continue | If | Else | Switch | Case | Default | For | While | Goto | Do | EQ | AMPAMP | PIPEPIPE | DASH | DASHDASH | DASHGT | PLUS | PLUSPLUS | SLASH | PERCENT @@ -152,9 +199,16 @@ impl Getter for ObjcCode { | CARETEQ | PIPEEQ | LBRACK | LBRACE | QMARK | PrimitiveType | TypeSpecifier | Sizeof | Signed | Unsigned | Long | Short | In | AT | ATtry | ATcatch | ATfinally | ATthrow | ATsynchronized | ATautoreleasepool | ATselector | ATencode | Struct - | Union | Enum | Typedef | ATinterface | ATimplementation | ATprotocol => { - TokenRole::Operator - } + | Union | Enum | Typedef | ATinterface | ATimplementation | ATprotocol | Static + | Extern | Register | Auto | Inline | Inline2 | Inline3 | Forceinline | ThreadLocal + | Thread | Const | Constexpr | Volatile | Restrict | Restrict2 | Atomic | Noreturn + | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension | Const2 | Complex + | Complex2 | Nullable | Nullable2 | Nullable3 | NullableResult | NullUnspecified + | Nonnull2 | Autoreleasing | Block | Strong | UnsafeUnretained | Weak | Bridge + | BridgeRetained | BridgeTransfer | Kindof | Covariant | Contravariant | Out + | Inout | Bycopy | Byref | Oneway | ATpublic | ATprivate | ATprotected | ATpackage + | AToptional | ATrequired => TokenRole::Operator, + Identifier if is_property_attribute_name(node, ancestors) => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only // classified node in a character literal, so it bills one diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 89f4cec4..567341c0 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -791,7 +791,18 @@ tokens involved the same way they spell real operators: placeholder type and stays unbilled. `extern "C"` bills `extern` and keeps `"C"` an operand. Attributes, the MS calling conventions and the Gecko `MOZ_*` macros bill nothing, and `override` and `final` - used as names (`int override = 1;`) stay operands. + used as names (`int override = 1;`) stay operands. #1579 extended this + to Objective-C, which bills C's list as C does plus its own + modifiers: the `@public`, `@private`, `@protected` and `@package` + ivar labels, the `@optional` and `@required` protocol sections, the + ownership qualifiers (`__weak`, `__strong`, `__block`, …), the + nullability qualifiers (`nullable`, `_Nullable`, `__nullable`, …), + `__kindof`, the `__covariant` and `__contravariant` variance, and the + `in`, `out`, `inout`, `bycopy`, `byref` and `oneway` method-type + qualifiers. A property attribute (`nonatomic`, `copy`, `getter`) bills + as its word; a `getter=` or `setter=` value stays an operand. The + Apple SDK macros (`NS_INLINE`, `FOUNDATION_EXPORT`, `IBOutlet`) bill + nothing, as the `MOZ_*` macros do. - **Angle brackets that enclose types are one bracket pair.** The `<` and `>` around type arguments and type parameters count as one `<>` operator, billed at the `<` as `()` is at the `(`, and the `>` counts diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 74239250..96f57395 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -608,6 +608,7 @@ mod tests { feature = "java", feature = "kotlin", feature = "mozcpp", + feature = "objc", feature = "typescript" ))] #[track_caller] @@ -7110,7 +7111,7 @@ end", /// name, one fixture line apiece, so dropping any one kind from an /// arm fails its own entry. The row tests above pin the counts; this /// one pins the spellings they leave out. - #[cfg(any(feature = "c", feature = "cpp", feature = "mozcpp"))] + #[cfg(any(feature = "c", feature = "cpp", feature = "mozcpp", feature = "objc"))] #[track_caller] fn assert_modifier_spellings_bill(file: &str, rows: &[(&str, &str)]) { for (spelling, source) in rows { @@ -7130,7 +7131,7 @@ end", /// The spellings C and C++ share, each in a declaration both /// grammars parse without error. - #[cfg(any(feature = "c", feature = "cpp", feature = "mozcpp"))] + #[cfg(any(feature = "c", feature = "cpp", feature = "mozcpp", feature = "objc"))] const C_FAMILY_MODIFIER_SPELLINGS: &[(&str, &str)] = &[ ("static", "static int a;\n"), ("extern", "extern int a;\n"), @@ -7207,6 +7208,203 @@ end", assert_operator_rows::("foo.m", ROWS); } + /// #1579: ObjC bills C's declaration modifiers as `c.rs` does + /// (#1578), and its own: ivar visibility labels, property attribute + /// words, ARC ownership and nullability qualifiers, protocol section + /// labels, method-type qualifiers and generic variance. Before the + /// fix none of them billed. + #[cfg(feature = "objc")] + #[test] + fn objc_modifiers_are_operators_1579() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `static`, `const`, `int`, `=`, `;`: the issue's first line + ("C modifiers", "static const int a = 1;\n", [5, 5]), + // `@interface`, `{}`, `@public`, `@private`, `int` x2, `;` x2 + ( + "ivar visibility", + "@interface C { @public int x; @private int y; }\n@end\n", + [6, 8], + ), + // `@interface`, `()`, `nonatomic`, `,` x2, `strong`, + // `getter`, `=`, `;`: `isY` is an operand, `id` and + // `@property` bill nothing + ( + "property attributes", + "@interface C\n@property (nonatomic, strong, getter=isY) id y;\n@end\n", + [8, 9], + ), + // `__weak`, `__strong`, `__block`, `int`, `;` x3 + ( + "ownership qualifiers", + "__weak id a;\n__strong id b;\n__block int c;\n", + [5, 7], + ), + // `*` x2, `_Nullable`, `nullable`, `;` x2 + ( + "nullability qualifiers", + "NSString * _Nullable f;\nNSString * nullable k;\n", + [4, 6], + ), + // `@protocol`, `@optional`, `@required`, `-` x2, `()` x3, + // `void` x2, `;` x2, `oneway`, `:`, `out`, `*` + ( + "protocol sections and method qualifiers", + "@protocol P\n@optional\n- (void)m;\n@required\n\ + - (oneway void)n:(out id *)b;\n@end\n", + [11, 16], + ), + // `@interface`, `<>`, `__covariant`, `:` + ( + "generic variance", + "@interface G<__covariant T> : NSObject\n@end\n", + [4, 4], + ), + ]; + assert_operator_rows::("foo.m", ROWS); + } + + /// #1579: the `nullable` qualifier and the `nullable` property + /// attribute are one operator. The attribute has no keyword kind, so + /// it is keyed by text, and keying the qualifier by kind would list + /// the one word twice in `n1`. + #[cfg(feature = "objc")] + #[test] + fn objc_nullable_is_one_operator_1579() { + const SOURCE: &str = "@interface C\n@property (nullable) id y;\n- (nullable id)m;\n@end\n"; + // `@interface`, `()` x2, `nullable` x2, `-`, `;` x2 + assert_operator_rows::("foo.m", &[("nullable", SOURCE, [5, 8])]); + let ops = ops_of::(SOURCE, "foo.m"); + assert_eq!(ops.operators, ["()", "-", ";", "@interface", "nullable"]); + assert_eq!(ops.operands, ["C", "m", "y"]); + } + + /// #1579's oracle: a `.m` is a C translation unit, so C's + /// modifier-heavy declarations score alike in both grammars, + /// vocabulary and multiplicities. Neither expectation is typed by + /// the fix: a modifier one getter drops shows as a disagreement. + #[cfg(all(feature = "c", feature = "objc"))] + #[test] + fn objc_c_modifiers_match_c_1579() { + let source = "static inline const volatile int a = 1;\nextern int b;\n\ + register int c;\nauto int d;\nint * restrict p;\n_Atomic int g;\n\ + __thread int t;\n_Noreturn void h(void);\n_Alignas(8) int m;\n"; + let objc = ops_of::(source, "foo.m"); + let c = ops_of::(source, "foo.c"); + assert_eq!(objc.operators, c.operators); + assert_eq!(objc.operands, c.operands); + // `static`, `inline`, `const`, `volatile`, `int` x8, `=`, `;` x9, + // `extern`, `register`, `auto`, `*`, `restrict`, `_Atomic`, + // `__thread`, `_Noreturn`, `void` x2, `()` x2, `_Alignas` + let rows = [("C modifiers", source, [18, 35])]; + assert_operator_rows::("foo.m", &rows); + assert_operator_rows::("foo.c", &rows); + } + + /// #1579: every modifier word the ObjC arm lists bills under its own + /// name, one fixture apiece, so dropping any kind from the arm fails + /// its own entry. The spellings C shares with C++ run through the + /// #1578 table; `OBJC_ONLY` holds the rest, C's `auto` included, plus + /// a sample of the text-keyed property attributes. + #[cfg(feature = "objc")] + #[test] + fn objc_modifier_spellings_bill_1579() { + const OBJC_ONLY: &[(&str, &str)] = &[ + ("auto", "auto int a;\n"), + ("__const", "__const int a;\n"), + ("_Complex", "double _Complex z;\n"), + ("__complex", "double __complex z;\n"), + ("nullable", "NSString * nullable p;\n"), + ("_Nullable", "NSString * _Nullable p;\n"), + ("__nullable", "NSString * __nullable p;\n"), + ("_Nullable_result", "NSString * _Nullable_result p;\n"), + ("_Null_unspecified", "NSString * _Null_unspecified p;\n"), + ("__nonnull", "NSString * __nonnull p;\n"), + ("__autoreleasing", "__autoreleasing id a;\n"), + ("__block", "__block int a;\n"), + ("__strong", "__strong id a;\n"), + ("__unsafe_unretained", "__unsafe_unretained id a;\n"), + ("__weak", "__weak id a;\n"), + ("__bridge", "void f(void) { id x = (__bridge id)p; }\n"), + ( + "__bridge_retained", + "void f(void) { void *x = (__bridge_retained void *)p; }\n", + ), + ( + "__bridge_transfer", + "void f(void) { id x = (__bridge_transfer id)p; }\n", + ), + ("__kindof", "__kindof NSView *v;\n"), + ( + "__covariant", + "@interface G<__covariant T> : NSObject\n@end\n", + ), + ( + "__contravariant", + "@interface G<__contravariant T> : NSObject\n@end\n", + ), + ("in", "@protocol P\n- (void)m:(in id)a;\n@end\n"), + ("out", "@protocol P\n- (void)m:(out id *)a;\n@end\n"), + ("inout", "@protocol P\n- (void)m:(inout id *)a;\n@end\n"), + ("bycopy", "@protocol P\n- (void)m:(bycopy id)a;\n@end\n"), + ("byref", "@protocol P\n- (void)m:(byref id)a;\n@end\n"), + ("oneway", "@protocol P\n- (oneway void)m;\n@end\n"), + ("@public", "@interface C { @public int x; }\n@end\n"), + ("@private", "@interface C { @private int x; }\n@end\n"), + ("@protected", "@interface C { @protected int x; }\n@end\n"), + ("@package", "@interface C { @package int x; }\n@end\n"), + ("@optional", "@protocol P\n@optional\n- (void)m;\n@end\n"), + ("@required", "@protocol P\n@required\n- (void)m;\n@end\n"), + ("copy", "@interface C\n@property (copy) id y;\n@end\n"), + ( + "readonly", + "@interface C\n@property (readonly) id y;\n@end\n", + ), + ("class", "@interface C\n@property (class) id y;\n@end\n"), + ( + "setter", + "@interface C\n@property (setter=sY:) id y;\n@end\n", + ), + ]; + assert_modifier_spellings_bill::("foo.m", C_FAMILY_MODIFIER_SPELLINGS); + assert_modifier_spellings_bill::("foo.m", OBJC_ONLY); + } + + /// #1579: the plain-word modifiers stay operands where they name a + /// variable, and a property attribute's `getter=` value is the + /// selector it names, not a modifier. The arms carry no guard on + /// these words, so this pins the grammar: it emits an `identifier` + /// for each name, and only the first word of a `property_attribute` + /// bills. + #[cfg(feature = "objc")] + #[test] + fn objc_modifier_words_as_names_stay_operands_1579() { + assert_keywords_are_operands_only::( + "int out = 1;\nint copy = 2;\nint oneway = 3;\nint strong = out + copy;\n", + "foo.m", + &["out", "copy", "oneway", "strong"], + ); + assert_keywords_are_operands_only::( + "@interface C\n@property (getter=atomic, setter=weak:) id y;\n@end\n", + "foo.m", + &["atomic", "weak"], + ); + } + + /// #1579: the Apple SDK macros the grammar files as storage classes + /// and qualifiers bill nothing, as Mozcpp's `MOZ_*` macros do: each + /// annotates a declaration, and a `.c` parses the spelling as a name. + #[cfg(feature = "objc")] + #[test] + fn objc_sdk_macros_bill_nothing_1579() { + // `int` x4, `;` x4, `()`, `void`, `{}`, `return`: the macros add + // nothing, and the declared names and `1` are the operands + const SOURCE: &str = "NS_INLINE int f(void) { return 1; }\nFOUNDATION_EXPORT int g;\n\ + IBOutlet int h;\n__unused int u;\n"; + let ops = ops_of::(SOURCE, "foo.m"); + assert_eq!(ops.operators, ["()", ";", "int", "return", "void", "{}"]); + assert_eq!(ops.operands, ["1", "f", "g", "h", "u"]); + } + /// #1556: Kotlin's `interface` bills as its `class` and `object` /// do. The modifier rows score one above the bare keyword since /// #1558: `enum` in `enum class` is a `class_modifier` like `data` From 456c349b22cc482974b0197d2db8087e5bf611c7 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:44:40 -0700 Subject: [PATCH 13/34] fix(perl): score a bareword key as a string tree-sitter-perl 1.1.2 parses every auto-quoted bareword key as a call: `(name => 1)` and `new(name => $x)` scored an ABC branch per key, and `(-name => 1)` / `$h{-name}` also billed a Halstead `-` operator and the unsigned operand `name`. Perl quotes a word before `=>`, and alone in a hash subscript, so each holds a string. lang_helpers::perl::perl_bareword_is_key now decides every bareword key shape through one key-position test, which #1545's file-test keys share. `=>` quotes the word right before it whatever expression it closes, so the test climbs while the key is its parent's last child: `(1 - name => 2)` quotes `name` too. ABC's bareword-call arm and the Halstead getter consult it; a signed key's `-` bills nothing and the key is spelled `-name`. Package-qualified words, words with arguments and bareword values keep their call. Cyclomatic and cognitive were never affected. Adds the abc/perl-bareword-keys width probe to the scaling gate. Fixes #1572 --- big-code-analysis-ast/src/checker.rs | 4 +- big-code-analysis-ast/src/getter/perl.rs | 24 +- .../src/lang_helpers/perl.rs | 172 ++++++++++-- big-code-analysis-bench/src/shapes.rs | 45 +++ big-code-analysis-book/src/metrics.md | 1 + docs/development/benchmarking.md | 1 + src/metrics/abc/perl.rs | 13 +- src/metrics/perl_keyword_key_tests.rs | 260 +++++++++++++++++- src/ops.rs | 26 ++ 9 files changed, 495 insertions(+), 51 deletions(-) diff --git a/big-code-analysis-ast/src/checker.rs b/big-code-analysis-ast/src/checker.rs index 00159000..2a1044ea 100644 --- a/big-code-analysis-ast/src/checker.rs +++ b/big-code-analysis-ast/src/checker.rs @@ -491,7 +491,9 @@ pub trait Checker { /// site: in `print shift;` the wrapper /// `call_expression_with_spaced_args` is the branch and the /// `print` bareword it holds is skipped, so that bareword is an - /// `is_call` match with no branch behind it. + /// `is_call` match with no branch behind it. So is an auto-quoted + /// hash key (`(name => 1)`, `$h{-name}`), which the grammar parses + /// as a bareword call and ABC reads as the string it is. /// /// The two sets coincide only in C, Objective-C, Go, Python, Lua /// and Bash, which have neither a separate construction syntax nor diff --git a/big-code-analysis-ast/src/getter/perl.rs b/big-code-analysis-ast/src/getter/perl.rs index 12228f27..9388c6c6 100644 --- a/big-code-analysis-ast/src/getter/perl.rs +++ b/big-code-analysis-ast/src/getter/perl.rs @@ -3,7 +3,7 @@ use super::*; use crate::lang_helpers::perl::{ - perl_and_is_operator, perl_dash_signs_keyword_key, perl_file_test_is_dash_key, + perl_and_is_operator, perl_bareword_key_sign, perl_dash_signs_key, perl_file_test_is_dash_key, perl_is_dash_key_word, perl_key_is_signed, perl_not_is_key, }; @@ -258,6 +258,11 @@ impl Getter for PerlCode { /// keys `(- not => 1)` as `"-not"` too. The sign is read off the tree /// (`perl_key_is_signed`), not the byte before the key, which can /// equally be an infix `-` or the end of a comment. + /// + /// Any other `-bareword` key (`(-name => 1)`, `$h{-name}`) bills its + /// word spelled from the sign (#1572). That is a source slice, so a + /// sign spaced off its word (`(- name => 1)`, rare) keeps the space: + /// one operand, distinct from `name`, but not merged with `-name`. fn get_operand_id<'a>( node: &Node<'a>, code: &'a [u8], @@ -274,6 +279,8 @@ impl Getter for PerlCode { } Perl::And if perl_key_is_signed(node, ancestors) => b"-and", Perl::Not if perl_key_is_signed(node, ancestors) => b"-not", + Perl::Identifier => perl_bareword_key_sign(node, ancestors) + .map_or(&code[start..end], |dash| &code[dash.start_byte()..end]), _ => &code[start..end], } } @@ -342,8 +349,11 @@ fn auto_quoted_key_role<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> Op // one operand spelling the whole key, and the word inside // bills nothing. A `-and` / `-not` key splits instead into // a `-` and the keyword, which the arms above bill as the - // key; the `-` is part of it and bills nothing. See - // `perl_file_test_is_dash_key`. + // key, and any other `-bareword` key (`(-name => 1)`) into a + // `-` and a bareword call, whose `identifier` bills the key + // (#1572, the local workaround for upstream #1571); in both + // the `-` is part of the key and bills nothing. See + // `perl_file_test_is_dash_key` and `perl_bareword_is_key`. P::FileHandleOperator if perl_file_test_is_dash_key(node, ancestors) => TokenRole::Operand, P::Identifier if ancestors @@ -353,13 +363,7 @@ fn auto_quoted_key_role<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> Op { TokenRole::Unknown } - P::DASH - if ancestors - .parent(node) - .is_some_and(|parent| perl_dash_signs_keyword_key(node, &parent)) => - { - TokenRole::Unknown - } + P::DASH if perl_dash_signs_key(node, ancestors) => TokenRole::Unknown, _ => return None, }; Some(role) diff --git a/big-code-analysis-ast/src/lang_helpers/perl.rs b/big-code-analysis-ast/src/lang_helpers/perl.rs index de72ed6c..c7494632 100644 --- a/big-code-analysis-ast/src/lang_helpers/perl.rs +++ b/big-code-analysis-ast/src/lang_helpers/perl.rs @@ -1,5 +1,6 @@ -//! Perl: telling the `and` and `not` operators, and the file tests, -//! from an auto-quoted key that the grammar lexes as one of them. +//! Perl: telling the `and` and `not` operators, the file tests, the +//! unary minus and bareword calls from an auto-quoted key that the +//! grammar lexes as one of them. use crate::Perl; use crate::node::{Ancestors, Node}; @@ -114,10 +115,9 @@ pub fn perl_not_is_key<'a>(not: &Node<'a>, parent: &Node<'a>) -> bool { /// The subscript form of the second shape (`$h{-x}`) recovers into an /// `ERROR` that drops the `-x` token altogether, so nothing is left to /// bill. A key whose letter is no file-test letter (`-name`, `-height`) -/// is a third shape, a `-` applied to the bareword, which this does not -/// cover: it still bills the `-` and the word, and ABC counts the word -/// as a call, as every bareword key does. Remove this, and every call -/// site, once the pin carries the upstream fix. +/// is a third shape, a `-` applied to the bareword, which +/// [`perl_bareword_is_key`] covers. Remove this, and every call site, +/// once the pin carries the upstream fix. #[must_use] pub(crate) fn perl_file_test_is_dash_key<'a>( test: &Node<'a>, @@ -130,7 +130,11 @@ pub(crate) fn perl_file_test_is_dash_key<'a>( if operand.is_error() { operand.is_child(Perl::FatComma as u16) && operand.start_row() == test.start_row() } else { - is_glued_word(&operand, test) && is_key_position(test, ancestors) + is_glued_word(&operand, test) + && ancestors + .iter(test) + .next() + .is_some_and(|(key, above)| is_key_position(&key, above)) } }) } @@ -139,7 +143,7 @@ pub(crate) fn perl_file_test_is_dash_key<'a>( /// file test bills whole, so it bills nothing of its own. `ancestors` /// is `node`'s chain. #[must_use] -pub fn perl_is_dash_key_word<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { +pub(crate) fn perl_is_dash_key_word<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { ancestors.iter(node).next().is_some_and(|(test, above)| { test.kind_id() == Perl::FileHandleOperator as u16 && is_glued_word(node, &test) @@ -147,6 +151,80 @@ pub fn perl_is_dash_key_word<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) }) } +/// Whether `word`, a Perl `call_expression_with_bareword` whose +/// ancestor chain is `ancestors`, is an auto-quoted hash key rather than +/// a call. +/// +/// FIXME(#1571 upstream): tree-sitter-perl 1.1.2 parses every bareword +/// key as a call. Perl quotes a word of letters, digits and underscores +/// before `=>` (perlop, "Comma Operator") and alone in a hash subscript +/// (perldata, "Scalar value constructors"), a leading `-` included, so +/// `(name => 1)`, `(-name => 1)`, `$h{-name}` and `$h{-size}` hold the +/// strings `"name"`, `"-name"`, `"-name"` and `"-size"`, and call +/// nothing. Three shapes reach here: +/// +/// - the bareword itself before a `=>` (`$h{name}` is an `identifier` +/// with no call around it, so the subscript form never arises); +/// - the operand of a `-` whose `unary_expression` is the key — +/// before a `=>`, or alone in a subscript; +/// - the word of a `-bareword` key the grammar reads as a file test +/// (`-size`), which `perl_is_dash_key_word` decides. +/// +/// A package-qualified word (`Foo::Bar => 1`) is not quoted by `=>` +/// and stays a call, as does a word with arguments (`name() => 1`), +/// which the grammar wraps in a call of its own. Remove this, and every +/// call site, once the pin carries the upstream fix. +#[must_use] +pub fn perl_bareword_is_key<'a>(word: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + if perl_is_dash_key_word(word, ancestors) { + return true; + } + if !is_plain_bareword(word) { + return false; + } + match ancestors.iter(word).next() { + Some((unary, above)) if is_dash_unary(&unary) => is_key_position(&unary, above), + _ => is_key_position(word, ancestors), + } +} + +/// Whether `dash`, a Perl `-` token whose ancestor chain is +/// `ancestors`, is the sign of an auto-quoted key rather than a unary +/// minus: a `-and` / `-not` key ([`perl_dash_signs_keyword_key`]) or a +/// `-bareword` key ([`perl_bareword_is_key`]). A key's sign is part of +/// the string, so it is no operator. +#[must_use] +pub(crate) fn perl_dash_signs_key<'a>(dash: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + ancestors.iter(dash).next().is_some_and(|(parent, above)| { + perl_dash_signs_keyword_key(dash, &parent) + || (parent.kind_id() == Perl::UnaryExpression as u16 + && next_non_extra(&parent, dash).is_some_and(|word| is_plain_bareword(&word)) + && is_key_position(&parent, above)) + }) +} + +/// The `-` that signs the `-bareword` key `identifier` spells, or +/// `None` when `identifier`, a Perl `identifier` whose ancestor chain is +/// `ancestors`, is no such key's word. The key bills as one operand +/// spelled from the `-` to the word's end, so `(-name => 1)` bills +/// `-name`, the spelling of its quoted twin's contents. +#[must_use] +pub(crate) fn perl_bareword_key_sign<'a>( + identifier: &Node<'a>, + ancestors: Ancestors<'a, '_>, +) -> Option> { + let mut up = ancestors.iter(identifier); + let (_, word_ancestors) = up + .next() + .filter(|(word, _)| word.kind_id() == Perl::CallExpressionWithBareword as u16)?; + let (unary, _) = up.next()?; + let dash = unary + .child_by_field_name("operator") + .filter(|op| op.kind_id() == Perl::DASH as u16)?; + // `word` and `dash` are siblings, so they share an ancestry. + perl_dash_signs_key(&dash, word_ancestors).then_some(dash) +} + /// Whether `dash`, a Perl `-` token whose parent is `parent`, is the /// sign of an auto-quoted `-and` or `-not` key (`$h{-not}`, /// `(-and => 1)`), which the grammar splits into the `-` and the keyword @@ -215,26 +293,66 @@ fn is_glued_word(word: &Node, test: &Node) -> bool { && word.start_byte() == test.start_byte() + FILE_TEST_TOKEN_LEN } -// Whether the `unary_expression` around `test` is the whole key of a -// hash subscript, which the grammar wraps in a `binary_expression` with -// no other child but comments, or is followed by a `=>`. -fn is_key_position<'a>(test: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - let mut up = ancestors.iter(test).map(|(ancestor, _)| ancestor); - let mut key = up.next(); - let mut parent = up.next(); - if parent - .is_some_and(|p| p.kind_id() == Perl::BinaryExpression as u16 && non_extra_count(&p) == 1) - { - key = parent; - parent = up.next(); +// Whether `key`, whose ancestor chain is `ancestors`, is a key where +// Perl auto-quotes one: the whole of a hash subscript, which the grammar +// may wrap in a `binary_expression` with no other child but comments, +// or the last thing before a `=>`. Every auto-quoted key shape asks this +// one question, of the node that spans its whole key. +// +// `=>` quotes the word immediately to its left whatever expression that +// word closes (perlop, "Comma Operator"), so `(1 - name => 2)` subtracts +// the string `"name"`. The grammar nests such a word as the last operand +// of its expression, so the test climbs while the node is its parent's +// last child: the sibling after the first ancestor that has one is the +// token after the word. A key that stopped being the whole of its node +// on the way up is no subscript key (`$h{1 - name}` is not quoted). +fn is_key_position<'a>(key: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let mut node = *key; + let mut whole = true; + for (parent, _) in ancestors.iter(key) { + if whole + && matches!( + parent.kind_id().into(), + Perl::HashAccessVariableSimple | Perl::HashAccessVariable + ) + { + return true; + } + // As in `perl_not_is_key`, only comments are passed over: the + // grammar wraps a stray `,` (`(name, => 1)`) in an `ERROR` it + // marks as an extra, and that `,`, not the word, is then the + // token the `=>` quotes, which is nothing. + if let Some(next) = parent + .children_after(&node) + .find(|sibling| sibling.is_error() || !sibling.as_tree_sitter().is_extra()) + { + return next.kind_id() == Perl::FatComma as u16; + } + whole &= parent.kind_id() == Perl::BinaryExpression as u16 && non_extra_count(&parent) == 1; + node = parent; } - key.zip(parent).is_some_and(|(key, parent)| { - matches!( - parent.kind_id().into(), - Perl::HashAccessVariableSimple | Perl::HashAccessVariable - ) || next_non_extra(&parent, &key) - .is_some_and(|next| next.kind_id() == Perl::FatComma as u16) - }) + false +} + +// Whether `word` is a call of a bare word and nothing else: one +// `identifier`, no package qualifier, no arguments. Only such a word is +// auto-quoted. +fn is_plain_bareword(word: &Node) -> bool { + word.kind_id() == Perl::CallExpressionWithBareword as u16 + && word + .children() + .filter(|child| !child.as_tree_sitter().is_extra()) + .map(|child| child.kind_id()) + .eq([Perl::Identifier as u16]) +} + +// Whether `node` is a unary minus, the `unary_expression` a `-bareword` +// key whose letter is no file-test letter parses as. +fn is_dash_unary(node: &Node) -> bool { + node.kind_id() == Perl::UnaryExpression as u16 + && node + .child_by_field_name("operator") + .is_some_and(|op| op.kind_id() == Perl::DASH as u16) } fn non_extra_count(node: &Node) -> usize { diff --git a/big-code-analysis-bench/src/shapes.rs b/big-code-analysis-bench/src/shapes.rs index 4279b756..1926f1ba 100644 --- a/big-code-analysis-bench/src/shapes.rs +++ b/big-code-analysis-bench/src/shapes.rs @@ -503,6 +503,29 @@ pub fn perl_dash_keys(width: usize) -> String { ) } +/// Perl: `sub f { return g(foo => 1, -foo => 1, …); }` — one call whose +/// argument list holds `width` bareword keys, alternately plain and +/// signed. +/// +/// The width shape for the #1572 bareword-key workaround, which asks of +/// every bareword call, and of every `-` before one, whether a `=>` +/// follows it. Unlike [`perl_dash_keys`], whose `-foo` the grammar reads +/// as a file test, neither key here has a letter that is a file test, so +/// this reaches the `-` and bareword-call path instead. +#[must_use] +pub fn perl_bareword_keys(width: usize) -> String { + let keys: Vec<&str> = (0..width) + .map(|i| { + if i % 2 == 0 { + "name => 1" + } else { + "-name => 1" + } + }) + .collect(); + format!("sub f {{ return g({}); }}\n", keys.join(", ")) +} + /// Perl: `my @r = (/a/, /a/, …);` — one list of `width` bare matches. /// /// The width shape for the Perl ABC pattern-role check, which asks of @@ -1355,6 +1378,28 @@ pub const PROBES: &[Probe] = &[ ABC asks through the same helper. The reading is the \ operand count, which grows with the width.", }, + Probe { + name: "abc/perl-bareword-keys", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc, Metric::Halstead], + reading: |m| m.halstead.total_operands(), + }, + render: perl_bareword_keys, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1572's bareword-key workaround asks of every bareword \ + call (ABC) and of every `-` before one (Halstead) \ + whether a `=>` follows it, the question \ + `halstead/perl-dash-keys` pins for file-test keys. \ + Finding that `=>` by scanning the list from its start \ + would be quadratic in the key count, as #1545's first \ + form was. The reading is the operand count, which \ + grows with the width; the branch count would not, \ + since the fix leaves only `g` scoring one.", + }, Probe { name: "abc/perl-pattern-list", lang: LANG::Perl, diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 567341c0..bb3a0b3a 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -163,6 +163,7 @@ application would over-count. | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | | C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | | Perl | A comparison's right operand is not a condition, although the grammar parses it as one | tree-sitter-perl 1.1.2 lets a chaining comparison (`<`, `>`, `<=`, `>=`, `lt`, `gt`, `le`, `ge`, `==`, `!=`, `eq`, `ne`) take everything of lower precedence to its right as its operand, so `$a > $b ? 1 : 2` parses as `$a > ($b ? 1 : 2)` and `$a > $b && $c` as `$a > ($b && $c)`. The ternary's condition slot and the chain's first operand then held `$b`, which scored as a condition of its own: one above the JavaScript twin for each misparsed comparison. The walker recognises the shape and leaves that operand to the comparison, so both spellings score what the parenthesised `($a > $b) ? 1 : 2` and `($a > $b) && $c` do. An explicit `$a > ($b ? 1 : 2)` is unaffected. `<=>`, `cmp`, `=~` and `!~` do not chain and parse correctly. The workaround goes when the grammar is fixed upstream (#1273). | +| Perl | An auto-quoted hash key is a string, not a call | Perl quotes a bareword before `=>` and alone in a hash subscript, a leading `-` included, so `(name => 1)`, `new(name => $x)`, `(-name => 1)` and `$h{-name}` hold the strings `"name"` and `"-name"`. tree-sitter-perl 1.1.2 parses each word as a call, which scored a branch, and each `-name` as a negation, which billed a Halstead `-` operator and the operand `name`. Each now scores as its quoted twin `('name' => 1)` does, and Halstead bills the key as one operand spelled as written: `name`, `-name`. A package-qualified word (`Foo::Bar => 1`), which `=>` does not quote, a word with arguments (`name() => 1`) and a bareword value (`(a => foo)`) keep their call. The workaround goes when the grammar is fixed upstream (#1571, #1572). | #### Where a constructor call lands {#abc-constructor-attribution} diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index 50a7e088..ad11b8e7 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -155,6 +155,7 @@ Read it as follows. | `abc/wide-let-declarators` | JavaScript | width | ABC's structural `const` check per `let` declarator (#1277) | linear | | `loc/wide-cfg-test-mod` | Rust | width | the `exclude_tests` prune arm's row bookkeeping (#1417) | linear | | `halstead/perl-dash-keys` | Perl | width | the dash-key workaround's `=>` lookup per key (#1545) | linear | +| `abc/perl-bareword-keys` | Perl | width | the bareword-key workaround's `=>` lookup per key (#1572) | linear | | `halstead/kotlin-noinline-params` | Kotlin | width | the modifier recovery guard's next-token lookup (#1558) | linear | | `abc/perl-pattern-list` | Perl | width | ABC's pattern-role climb through `( … )` (#1467) | linear | | `cyclomatic/cpp-plus-chain` | C++ | depth | shape control for the three rows below | linear | diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index 0589a4c4..e958bbb9 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -14,7 +14,7 @@ use super::{ wrapped_operand, }; use crate::lang_helpers::perl::{ - perl_and_is_operator, perl_dash_signs_keyword_key, perl_is_dash_key_word, perl_not_is_key, + perl_and_is_operator, perl_bareword_is_key, perl_dash_signs_keyword_key, perl_not_is_key, }; use crate::*; @@ -566,10 +566,11 @@ impl Abc for PerlCode { // when this node is the outermost dispatch site. When the // bareword sits inside one of the wrappers above, the // outer node has already been counted and this child - // would double the branch tally. Nor is the word of a - // `-bareword` key a call (FIXME(#1545 upstream)): the - // grammar splits `$h{-foo}` into a file test on a bareword - // `oo`, and the key is a string. + // would double the branch tally. Nor is an auto-quoted + // key a call (FIXME(#1571 upstream)): the grammar reads + // `(name => 1)`, `(-name => 1)` and `$h{-name}` as calls of + // `name`, and `$h{-foo}` as a file test on a bareword `oo`, + // where Perl holds a string (#1545, #1572). P::CallExpressionWithBareword if !ancestors.parent(node).is_some_and(|p| { matches!( @@ -580,7 +581,7 @@ impl Abc for PerlCode { | P::CallExpressionWithVariable | P::CallExpressionRecursive ) - }) && !perl_is_dash_key_word(node, ancestors) => + }) && !perl_bareword_is_key(node, ancestors) => { stats.branches += 1.; } diff --git a/src/metrics/perl_keyword_key_tests.rs b/src/metrics/perl_keyword_key_tests.rs index e16500af..5b1dfed9 100644 --- a/src/metrics/perl_keyword_key_tests.rs +++ b/src/metrics/perl_keyword_key_tests.rs @@ -1,7 +1,8 @@ //! An auto-quoted `and` or `not` key scores like any other key, and the //! `and` and `not` operators still score as operators (#1539, #1541). //! The same holds for a `-bareword` key and the file tests the grammar -//! mistakes it for (#1545). +//! mistakes it for (#1545), and for every other bareword key, which the +//! grammar reads as a call (#1572). //! //! Perl quotes a bareword before `=>` and alone in a hash subscript, so //! `$h{and}` and `(not => 1)` hold a string key and no operator. @@ -416,10 +417,11 @@ const DASH_KEY_ROWS: [(&str, &str, [u64; 4], Halstead); 20] = [ [2, 1, 0, 0], [5, 5, 3, 4], ), + // The `ext` key scored a branch for "calling" it until #1572. ( - "f('-text' => 1, ext => 2);", + "f('-text' => 1, 'ext' => 2);", "f(-text => 1, ext => 2);", - [2, 2, 0, 0], + [2, 1, 0, 0], [6, 7, 5, 6], ), ( @@ -499,9 +501,9 @@ const DASH_KEY_ROWS: [(&str, &str, [u64; 4], Halstead); 20] = [ /// swallowing the `=>`, and `-not` as a `-` applied to the keyword. /// Before the fix each `-foo` billed the operand `oo` and an ABC branch /// for calling it, and each `-not` / `-and` billed a `-` operator. A key -/// whose letter is no file-test letter (`-name`) is not among them: it -/// parses as a `-` applied to a bareword, which still bills the `-` and -/// an ABC branch for the word, as every bareword key does. +/// whose letter is no file-test letter (`-name`) parses as a `-` applied +/// to a bareword instead, and is pinned by +/// `perl_bareword_key_scores_like_its_quoted_twin`. /// /// The rows pairing a key with a second key equal to the misparsed /// word (`ext`, `not`) or to itself (`-x` twice) pin the operand's @@ -530,7 +532,9 @@ fn perl_dash_key_scores_like_its_quoted_twin() { /// A `-and` / `-not` key keeps its sign when whitespace separates the /// two: Perl keys `(- not => 1)` as `"-not"`, so it must not share a /// vocabulary entry with the plain `not` key beside it. The twins' -/// values are `DASH_KEY_ROWS`' for the glued spelling. +/// values are `DASH_KEY_ROWS`' for the glued spelling. A `- name` key +/// keeps its sign too (#1572); it is spelled as written, space and all, +/// which the counts cannot see, since only one signed key is present. #[cfg(feature = "perl")] #[test] fn perl_spaced_dash_key_keeps_its_sign() { @@ -542,6 +546,10 @@ fn perl_spaced_dash_key_keeps_its_sign() { "f('-not' => 1, not => 2);", "f(- # c\n not => 1, not => 2);", ), + ( + "f('-name' => 1, 'name' => 2);", + "f(- name => 1, name => 2);", + ), ] { let want = measure_with_branches(twin); assert_eq!( @@ -642,3 +650,241 @@ fn perl_file_test_outside_a_key_keeps_its_reading() { ); } } + +/// `(quoted_twin, bareword_key, [cyclomatic, branches, conditions, +/// cognitive], halstead)` rows for +/// `perl_bareword_key_scores_like_its_quoted_twin`. Every value is the +/// twin's, measured on `main` before #1572. +#[cfg(feature = "perl")] +const BAREWORD_KEY_ROWS: [(&str, &str, [u64; 4], Halstead); 23] = [ + ( + "my %h = ('name' => 1); return 1;", + "my %h = (name => 1); return 1;", + [2, 0, 0, 0], + [8, 9, 4, 5], + ), + ( + "my %h = ('-name' => 1); return 1;", + "my %h = (-name => 1); return 1;", + [2, 0, 0, 0], + [8, 9, 4, 5], + ), + ( + "my %h; return $h{'-name'};", + "my %h; return $h{-name};", + [2, 0, 0, 0], + [6, 8, 4, 4], + ), + ( + "my %h; return $h{ '-name' };", + "my %h; return $h{ -name };", + [2, 0, 0, 0], + [6, 8, 4, 4], + ), + ( + "my $r; return $r->{'-name'};", + "my $r; return $r->{-name};", + [2, 0, 0, 0], + [7, 10, 3, 4], + ), + ( + "my %h; return @h{'-a'};", + "my %h; return @h{-a};", + [2, 0, 0, 0], + [5, 7, 4, 4], + ), + ( + "my %h; if ($h{'-name'}) { return 1; } return 0;", + "my %h; if ($h{-name}) { return 1; } return 0;", + [3, 0, 1, 1], + [8, 13, 6, 6], + ), + ( + "my $r = { 'a' => { 'b' => 1 } }; return $r;", + "my $r = { a => { b => 1 } }; return $r;", + [2, 0, 0, 0], + [8, 13, 5, 6], + ), + // A named-argument call scores its callee alone. + ( + "new('name' => $x, 'size' => 3);", + "new(name => $x, size => 3);", + [2, 1, 0, 0], + [7, 8, 6, 6], + ), + ( + "foo 'name' => 1;", + "foo name => 1;", + [2, 1, 0, 0], + [4, 4, 4, 4], + ), + ( + "$o->m('-name' => 1);", + "$o->m(-name => 1);", + [2, 1, 0, 0], + [7, 7, 5, 5], + ), + // Perl quotes a word across a comment or line break before `=>`. + ( + "my %h = ('name' # c\n => 1); return 1;", + "my %h = (name # c\n => 1); return 1;", + [2, 0, 0, 0], + [8, 9, 4, 5], + ), + // The spelling rows: a signed and a plain key are two operands, and + // two signed keys one, as their twins are. Billed as `name`, the + // first pair merges (n2 4) and the second still matches; billed as + // `-` + `name`, both also bill a `-` operator. + ( + "f('-name' => 1, 'name' => 2);", + "f(-name => 1, name => 2);", + [2, 1, 0, 0], + [6, 7, 5, 6], + ), + ( + "f('-name' => 1, '-name' => 2);", + "f(-name => 1, -name => 2);", + [2, 1, 0, 0], + [6, 7, 4, 6], + ), + // A builtin's name is quoted like any other word. + ( + "my %h = ('shift' => 1, 'time' => 2); return 1;", + "my %h = (shift => 1, time => 2); return 1;", + [2, 0, 0, 0], + [9, 11, 6, 7], + ), + ( + "my @a = ('a' => 'b' => 1); return @a;", + "my @a = (a => b => 1); return @a;", + [2, 0, 0, 0], + [8, 10, 5, 6], + ), + // A bareword *value* is no key, and still scores its call. + ( + "my @a = ('a' => foo); return @a;", + "my @a = (a => foo); return @a;", + [2, 1, 0, 0], + [8, 9, 4, 5], + ), + // `=>` quotes the word right before it whatever expression the word + // closes, which the grammar nests as that expression's last operand. + // A subtraction's `-` stays an operator. + ( + "my @a = (1 - 'name' => 2); return @a;", + "my @a = (1 - name => 2); return @a;", + [2, 0, 0, 0], + [9, 10, 5, 6], + ), + ( + "my @a = ($x . 'name' => 2); return @a;", + "my @a = ($x . name => 2); return @a;", + [2, 0, 0, 0], + [10, 11, 5, 6], + ), + ( + "my @a = (f(1), 'name' => 2); return @a;", + "my @a = (f(1), name => 2); return @a;", + [2, 1, 0, 0], + [9, 11, 5, 7], + ), + // After a keyword or file-test key, whose recovery leaves an + // `ERROR` in the same list. + ( + "f(not => 1, 'name' => 2);", + "f(not => 1, name => 2);", + [2, 1, 0, 0], + [6, 7, 5, 6], + ), + ( + "f(and => 1, '-name' => 2);", + "f(and => 1, -name => 2);", + [2, 1, 0, 0], + [6, 7, 5, 6], + ), + ( + "f('-x' => 1, 'name' => 2);", + "f(-x => 1, name => 2);", + [2, 1, 0, 0], + [6, 7, 5, 6], + ), +]; + +/// Each bareword key below scores exactly as its quoted twin does, in +/// all four metrics (#1572). tree-sitter-perl reads `name` before `=>` +/// as a call of `name`, and `-name` there or alone in a subscript as a +/// `-` applied to that call. Before the fix each key scored an ABC +/// branch, and each `-name` billed a `-` operator and the unsigned +/// operand `name`. Cyclomatic and cognitive were never affected, which +/// the decision columns pin. +#[cfg(feature = "perl")] +#[test] +fn perl_bareword_key_scores_like_its_quoted_twin() { + for (twin, key, decisions, halstead) in BAREWORD_KEY_ROWS { + assert_eq!(twin.replace('\'', ""), key, "`{key}` drifted from its twin"); + let want = measure_with_branches(twin); + assert_eq!( + want, + (decisions, halstead), + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure_with_branches(key), + want, + "`{key}` must score like `{twin}`" + ); + } +} + +/// A bareword that is no auto-quoted key keeps the call, and a `-` that +/// signs none keeps its operator: a word followed by anything but `=>`, +/// a package-qualified word (`=>` quotes only letters, digits and +/// underscores), a word with arguments, a call inside a subscript, a +/// `-bareword` that is only part of one, a word a stray `,` separates +/// from the `=>` (the grammar hides that `,` in an extra `ERROR`, and +/// the `=>` then quotes the `,`, which is nothing), and a unary minus on +/// a variable. Every value is `main`'s, measured before #1572. +#[cfg(feature = "perl")] +#[test] +fn perl_bareword_outside_a_key_keeps_its_call() { + let rows: [(&str, [u64; 4], Halstead); 11] = [ + ("foo(-name);", [2, 2, 0, 0], [5, 5, 3, 3]), + ( + "my @a = (name, => 1); return @a;", + [2, 1, 0, 0], + [9, 10, 4, 5], + ), + ( + "my @a = (-name, => 1); return @a;", + [2, 1, 0, 0], + [10, 11, 4, 5], + ), + ("return -name;", [2, 1, 0, 0], [5, 5, 2, 2]), + ("my @a = (name, 1); return @a;", [2, 1, 0, 0], [8, 9, 4, 5]), + ( + "my @a = (Foo::Bar => 1); return @a;", + [2, 1, 0, 0], + [9, 10, 5, 6], + ), + ( + "my @a = (name() => 1); return @a;", + [2, 1, 0, 0], + [8, 10, 4, 5], + ), + ("my %h; return $h{foo()};", [2, 1, 0, 0], [7, 9, 4, 4]), + ( + "my %h; return $h{-name . 'x'};", + [2, 1, 0, 0], + [8, 10, 5, 5], + ), + ("my %h; return $h{1 - name};", [2, 1, 0, 0], [7, 9, 5, 5]), + ("my $y = -$x; return $y;", [2, 0, 0, 0], [8, 11, 3, 4]), + ]; + for (body, decisions, halstead) in rows { + assert_eq!( + measure_with_branches(body), + (decisions, halstead), + "`{body}` changed its reading" + ); + } +} diff --git a/src/ops.rs b/src/ops.rs index 1e193dc8..c3385e1b 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -509,6 +509,32 @@ mod tests { ); } + /// A bareword key bills as the string it is (#1572): `-name` as one + /// operand with its sign, no `-` operator, and apart from the plain + /// `name` key. The expected vocabularies are the quoted twin's + /// (`'-name' => 1, 'name' => 2`, `$h{'-name'}`) with the quotes + /// dropped; before the fix `-name` billed `-` and folded into `name`. + /// The second fixture is the guard: a subtraction before a key and a + /// unary minus on a variable keep their `-`. + #[cfg(feature = "perl")] + #[test] + fn perl_bareword_keys_bill_as_written() { + check_ops( + LANG::Perl, + "my %h = (-name => 1, name => 2); my $v = $h{-name};", + "foo.pl", + &mut ["$", "()", ",", ";", "=", "=>", "my", "{}"], + &mut ["$h", "$v", "%h", "-name", "1", "2", "name"], + ); + check_ops( + LANG::Perl, + "my @a = (1 - size => 3, -$x);", + "foo.pl", + &mut ["()", ",", "-", ";", "=", "=>", "my", "$"], + &mut ["$x", "1", "3", "@a", "size"], + ); + } + #[cfg(feature = "python")] #[test] fn python_function_ops() { From cd26f65bb86d701ac297258d74dc956081255226 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 18:11:27 -0700 Subject: [PATCH 14/34] fix(elixir): score every section spelling alike Two spellings of one Elixir program scored differently in five places. A bracketed keyword list in last position (`if(c, [do: 1, else: 2])`) is the same argument as the bare `do:` form, but every section reader only recognised a bare `keywords` node. One lookup now finds the trailing keyword list in either spelling, and quoted keys (`"do":`), for `if` / `unless` else, `with`'s else tier, `cond`'s free `true ->` and the `for ... reduce:` head skip. `Kernel.||(a, b)` and `Kernel.&&(a, b)` now score ABC conditions and a cognitive boolean sequence through the predicate cyclomatic already used, and a nested call no longer pays a slot as an operand. A user module's `Foo.||` still scores nothing. Cognitive scored the keyword forms' `else:` / `rescue:` / `catch:` nothing; they now score as their blocks do, for `if`, `unless`, `try` and the `def` family. A `with`'s `else` adds nothing in either form: its clauses are the arms of a `case`, already paid by the container. Fixes #1574 --- .../src/lang_helpers/elixir.rs | 71 ++-- big-code-analysis-book/src/metrics.md | 13 +- src/metrics/abc.rs | 22 +- src/metrics/abc/elixir.rs | 140 ++++--- src/metrics/cognitive/elixir.rs | 103 +++++- src/metrics/cyclomatic/elixir.rs | 12 +- src/metrics/elixir_decisions.rs | 157 ++++++-- src/metrics/elixir_spelling_parity_tests.rs | 344 ++++++++++++++++++ src/metrics/mod.rs | 3 + 9 files changed, 740 insertions(+), 125 deletions(-) create mode 100644 src/metrics/elixir_spelling_parity_tests.rs diff --git a/big-code-analysis-ast/src/lang_helpers/elixir.rs b/big-code-analysis-ast/src/lang_helpers/elixir.rs index 5fff724a..f09ea577 100644 --- a/big-code-analysis-ast/src/lang_helpers/elixir.rs +++ b/big-code-analysis-ast/src/lang_helpers/elixir.rs @@ -98,7 +98,7 @@ pub fn elixir_do_block_call_children<'a>( /// recovery the parent can be an `ERROR` with no operands, so every /// decision metric keys on this parent rather than on the token (#1531, /// #1534). A remote call that names the operator still *applies* it, -/// though; see [`elixir_remote_call_applies`]. +/// though; see [`elixir_applying_remote_call`]. /// /// Matched by rule name rather than by `kind_id`: the grammar aliases /// `binary_operator` to three ids (`Elixir::BinaryOperator` through @@ -114,39 +114,56 @@ pub fn elixir_applying_operator<'a>( .filter(|parent| parent.kind() == "binary_operator") } -/// Whether `op`, an operator token, is the name of a remote call on -/// `Kernel` with arguments: `Kernel.||(a, b)`, or `x |> Kernel.||(y)`, -/// which the pipe rewrites to it. Such a call expands the `Kernel` macro -/// exactly as `a || b` does, so it short-circuits and decides as the -/// operator does, where a capture (`&Kernel.||/2`, a call with no -/// arguments) only names it. It has no `binary_operator`, so it carries no -/// `left` / `right` operands either. +/// The operator token a remote call on `Kernel` with arguments applies, +/// or `None` for any other node: the `||` of `Kernel.||(a, b)`, or of +/// `x |> Kernel.||(y)`, which the pipe rewrites to it. Such a call +/// expands the `Kernel` macro exactly as `a || b` does, so it +/// short-circuits and decides as the operator does, where a capture +/// (`&Kernel.||/2`, a call with no arguments) only names it. It has no +/// `binary_operator`, so its operands are the call's arguments, and under +/// a pipe the piped value, rather than `left` / `right` fields. /// /// The receiver must spell `Kernel` (or `Elixir.Kernel`): a user module /// may define `||/2` as an ordinary function, and `Foo.||(a, b)` neither /// short-circuits nor decides. /// -/// Matched by rule name: `dot` and `arguments` are aliased to several -/// ids each (grammar-dispatch §1). +/// Only an operator the grammar lexes as a token inside the call's +/// `operator_identifier` is found. `Kernel.and(a, b)` spells its name as +/// a plain `identifier`, so it is not; nor is the childless `..`. +/// +/// Matched by rule name: `call`, `dot` and `arguments` are aliased to +/// several ids each (grammar-dispatch §1). +#[must_use] +pub fn elixir_kernel_operator_token<'a>(call: &Node<'a>, code: &[u8]) -> Option> { + if call.kind() != "call" || !call.children().any(|child| child.kind() == "arguments") { + return None; + } + let dot = call.child_by_field_name("target").filter(|target| { + target.kind() == "dot" + && target.child_by_field_name("left").is_some_and(|receiver| { + matches!( + code.get(receiver.start_byte()..receiver.end_byte()), + Some(b"Kernel" | b"Elixir.Kernel") + ) + }) + })?; + dot.child_by_field_name("right") + .filter(|name| name.kind() == "operator_identifier")? + .child(0) +} + +/// The `Kernel` remote call that applies `op`, an operator token — the +/// call [`elixir_kernel_operator_token`] answers `op` for — or `None` +/// when `op` sits anywhere else. The token's parents are the +/// `operator_identifier`, then the `dot`, then the call. #[must_use] -pub fn elixir_remote_call_applies<'a>( +pub fn elixir_applying_remote_call<'a>( op: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>, -) -> bool { - let mut up = ancestors.iter(op).map(|(node, _)| node); - up.next() - .is_some_and(|name| name.kind() == "operator_identifier") - && up.next().is_some_and(|dot| { - dot.kind() == "dot" - && dot.child_by_field_name("left").is_some_and(|receiver| { - matches!( - code.get(receiver.start_byte()..receiver.end_byte()), - Some(b"Kernel" | b"Elixir.Kernel") - ) - }) - }) - && up.next().is_some_and(|call| { - call.kind() == "call" && call.children().any(|child| child.kind() == "arguments") - }) +) -> Option> { + let (call, _) = ancestors.iter(op).nth(2)?; + elixir_kernel_operator_token(&call, code) + .is_some_and(|token| token.id() == op.id()) + .then_some(call) } diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index bb3a0b3a..7111a61b 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -158,7 +158,7 @@ application would over-count. | Kotlin | A subject-less `when` arm scores through the predicate slot; a subject-ful arm scores per entry | A subject-less arm's condition (`when { x > 5 -> … }`) is an ordinary boolean expression, compiled as an `if` predicate, so the comparison inside it is already counted by the operator arms and a blanket per-entry increment double-counted it. A subject-ful arm (`when (x) { in 1..2 -> … }`) lists a pattern rather than an independent boolean expression, so the implicit `subject == pattern` is a decision nothing in the source spells and the entry itself pays for it (#1421). | | Ruby | A subject-less `case` scores each `when` pattern as an `\|\|` operand; a subject-ful `case` scores per clause | A subject-less clause (`case; when x > 5`) lists ordinary boolean expressions, evaluated like an `if` predicate, and several patterns (`when a, b`) are an implicit `\|\|`. Each pattern therefore scores once unless its own operator already counted it, so `when x > 5` scores 1 like `if x > 5`, and `when a, b` scores 2 like `if a \|\| b`. Before this, a blanket per-clause increment counted a comparison twice. A subject-ful clause (`case x; when 1`) matches each pattern with an implicit `pattern === x` that nothing in the source spells, so the clause pays once (#1453). | | Go | A tagless `switch` scores each case expression as an `\|\|` operand; a tagged `switch` scores per case | A tagless switch (`switch { case x > 5: }`, with or without an init statement) is `switch true`: each case expression is an ordinary boolean scored by the same slot an `if` predicate uses, and a list (`case a, b:`) is an implicit `\|\|`. So `case x > 5:` scores 1 like `if x > 5`, and `case a, b:` scores 2 like `if a \|\| b`. Before this, a blanket per-case increment counted a comparison twice. A tagged case (`switch x { case 1: }`) compares with an implicit `x == 1` that nothing in the source spells, so the case pays once; type-switch and `select` cases are unchanged (#1523). | -| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive` (its `after` timeout included), of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). | +| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive` (its `after` timeout included), of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). A bracketed keyword list in last position (`if(c, [do: 1, else: 2])`) is the same argument as the bare one and scores the same, as does a quoted key (`"else": 2`); a list anywhere else, or one with a non-pair item, holds no sections. A `Kernel.&&` / `Kernel.\|\|` remote call scores its operands as `a && b` / `a \|\| b` does, the piped value of `a \|> Kernel.\|\|(b)` included (#1574). | | C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these brackets as an operator (a type-enclosing pair as one `<>` since #1559 and #1567), and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | | C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | @@ -374,7 +374,13 @@ Sonar ecosystem. nesting-bearing constructs it can identify and omits the recursion (B3) and unstructured-jump (B2) increments that the SonarSource specification adds for languages that expose those - shapes syntactically. + shapes syntactically. A `with`'s `else` adds nothing: its clauses + are the arms of the `case` the construct is in all but spelling, + already paid by the `with`'s own +nesting, as Ruby's `case`/`else` + is. An `if` / `unless` / `try` `else` still adds +1. The keyword + forms (`else:`, `rescue:`, `catch:`, bare or bracketed) score as + their `do … end` blocks do, and `Kernel.||(a, b)` is a boolean + sequence like `a || b` (#1574). - For every language with a syntactic function-definition node, a nested function — a local function, or a method on a local / inner class — **resets the nesting counter to zero** at its @@ -899,7 +905,8 @@ tokens involved the same way they spell real operators: `&foo/2`, so the token inside it adds no operator; `&==/2` scores what `&foo/2` does. A captured `&&` / `||` / `and` / `or` is no decision either (#1534), but the remote call applies the operator, - so `Kernel.||(a, b)` is a cyclomatic decision as `a || b` is. This + so `Kernel.||(a, b)` scores what `a || b` does in cyclomatic, ABC and + cognitive alike, and a user module's `Foo.||(a, b)` in none (#1574). This differs from a C# `operator <`, whose `<` has no wrapper and so stays the operator the rule above makes it. diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index f16f646c..fa690ff6 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -501,7 +501,7 @@ pub(super) fn peel<'a>(node: &Node<'a>, step: PeelStep<'a>) -> (Node<'a>, bool) pub(super) fn count_boolean_slot<'a>( slot: &Node<'a>, step: PeelStep<'a>, - scores_itself: fn(&Node<'a>) -> bool, + scores_itself: impl Fn(&Node<'a>) -> bool, conditions: &mut f64, ) { if !scores_itself(&peel(slot, step).0) { @@ -527,16 +527,18 @@ pub(super) fn count_negated_operand<'a>( } } -// A language's scorer for one operand: its `count_boolean_slot` or -// `count_negated_operand` wrapper. -pub(super) type OperandScorer = fn(&Node, &mut f64); - // Fitzpatrick Rule 9 for a chain whose grammar names its two operands -// `left` and `right`: each is scored by `score`. Read by field rather +// `left` and `right`: each is scored by `score`, the language's +// `count_boolean_slot` or `count_negated_operand` wrapper — a closure +// where it needs the source bytes (Elixir's, #1574). Read by field rather // than as every operand child, so a comment beside the operator — a // named child too — does not pay. The caller owns the gate deciding // that `chain` is a chain at all. -pub(super) fn count_field_operands(chain: &Node, score: OperandScorer, conditions: &mut f64) { +pub(super) fn count_field_operands( + chain: &Node, + score: impl Fn(&Node, &mut f64), + conditions: &mut f64, +) { for field in ["left", "right"] { if let Some(operand) = chain.child_by_field_name(field) { score(&operand, conditions); @@ -547,7 +549,11 @@ pub(super) fn count_field_operands(chain: &Node, score: OperandScorer, condition // Each operand child of `list` — an argument list, a `return` value // list, a chain whose grammar names no operand field — scored by // `score`. `is_operand` keeps punctuation and comments out. -pub(super) fn count_each_operand(list: &Node, score: OperandScorer, conditions: &mut f64) { +pub(super) fn count_each_operand( + list: &Node, + score: impl Fn(&Node, &mut f64), + conditions: &mut f64, +) { for operand in list.children().filter(is_operand) { score(&operand, conditions); } diff --git a/src/metrics/abc/elixir.rs b/src/metrics/abc/elixir.rs index 300eefa4..c0749991 100644 --- a/src/metrics/abc/elixir.rs +++ b/src/metrics/abc/elixir.rs @@ -9,11 +9,17 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_field_operands, last_operand, wrapped_operand}; -use crate::lang_helpers::elixir::{elixir_applying_operator, elixir_call_keyword}; +use super::{ + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, last_operand, + wrapped_operand, +}; +use crate::lang_helpers::elixir::{ + elixir_applying_operator, elixir_applying_remote_call, elixir_call_keyword, + elixir_kernel_operator_token, +}; use crate::metrics::elixir_decisions::{ - elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_sole_unguarded_pattern, - elixir_when_alternative, elixir_when_is_guard, + elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_keyword_section, + elixir_sole_unguarded_pattern, elixir_when_alternative, elixir_when_is_guard, }; use crate::*; @@ -71,33 +77,37 @@ fn elixir_binary_operator_is(node: &Node, accept: impl Fn(Elixir) -> bool) -> bo // arms) and for an `&&` / `||` / `and` / `or` chain, whose walker scores // each operand through `elixir_count_condition` — so every operand // either pays there or is one of these, and a chain always carries at -// least one condition of its own. +// least one condition of its own. A short-circuit `Kernel` remote call +// (`Kernel.||(a, b)`) is that chain spelled as a call, and its operands +// pay through `elixir_count_remote_chain` in the same way (#1574). // // The operator list must agree with the token arms in `compute`: an // operator listed here and counted nowhere would leave its slot at zero, // and one counted there but missing here would pay twice. -fn elixir_condition_scores_itself(expr: &Node) -> bool { +fn elixir_condition_scores_itself(expr: &Node, code: &[u8]) -> bool { use Elixir as E; - elixir_binary_operator_is(expr, |op| { - matches!( - op, - E::EQEQ - | E::EQEQEQ - | E::BANGEQ - | E::BANGEQEQ - | E::LTEQ - | E::GTEQ - | E::LT - | E::GT - | E::In - | E::Notin - | E::AMPAMP - | E::PIPEPIPE - | E::And - | E::Or - ) - }) + elixir_kernel_operator_token(expr, code) + .is_some_and(|op| matches!(op.kind_id().into(), E::AMPAMP | E::PIPEPIPE)) + || elixir_binary_operator_is(expr, |op| { + matches!( + op, + E::EQEQ + | E::EQEQEQ + | E::BANGEQ + | E::BANGEQEQ + | E::LTEQ + | E::GTEQ + | E::LT + | E::GT + | E::In + | E::Notin + | E::AMPAMP + | E::PIPEPIPE + | E::And + | E::Or + ) + }) } // Scores one boolean slot — an `if` / `unless` predicate, a `cond` @@ -115,11 +125,11 @@ fn elixir_condition_scores_itself(expr: &Node) -> bool { // whether the decision is already paid for, rather than whether the // occupant is a known terminal, also scores what the list never held — // `a && x + 1`, `a && @flag`, `a && -x` each scored one below `a && b`. -fn elixir_count_condition(condition: &Node, conditions: &mut f64) { +fn elixir_count_condition(condition: &Node, code: &[u8], conditions: &mut f64) { count_boolean_slot( condition, elixir_wrapper_operand, - elixir_condition_scores_itself, + |expr| elixir_condition_scores_itself(expr, code), conditions, ); } @@ -139,9 +149,9 @@ fn elixir_count_condition(condition: &Node, conditions: &mut f64) { // `binary_operator` has no `right` child, which the grammar declares // required — so the `if let`'s else is unreachable at the pin rather // than untested. -fn elixir_count_guard(when_operator: &Node, conditions: &mut f64) { +fn elixir_count_guard(when_operator: &Node, code: &[u8], conditions: &mut f64) { if let Some(alternative) = elixir_when_alternative(when_operator) { - elixir_count_condition(&alternative, conditions); + elixir_count_condition(&alternative, code, conditions); } } @@ -163,33 +173,58 @@ fn elixir_count_guard(when_operator: &Node, conditions: &mut f64) { // clause (`x when g ->`, `kind, reason ->`) pays one for its pattern // match, and a guard pays its own slot through the `when` arm, as the // two `Cyclomatic` decisions it carries. -fn elixir_count_clause(clause: &Node, conditions: &mut f64) { +fn elixir_count_clause(clause: &Node, code: &[u8], conditions: &mut f64) { match elixir_sole_unguarded_pattern(clause) { - Some(pattern) => elixir_count_condition(&pattern, conditions), + Some(pattern) => elixir_count_condition(&pattern, code, conditions), None => *conditions += 1., } } -// Whether an `if` / `unless` Call carries an `else` branch, in either +// Whether an `if` / `unless` Call carries an `else` branch, in any // spelling: the block form's `else_block` inside the `do_block` -// (`if b do … else … end`), or the keyword form's `else:` pair -// (`if b, do: …, else: …`). The keyword token's text carries its colon -// and the whitespace after it, so it is trimmed before comparing. +// (`if b do … else … end`), or the keyword form's `else:` pair, bare +// (`if b, do: …, else: …`) or bracketed (`if(b, [do: …, else: …])`). fn elixir_has_else(call: &Node, code: &[u8]) -> bool { use Elixir as E; call.children().any(|child| match child.kind_id().into() { E::DoBlock => child.children().any(|c| c.kind_id() == E::ElseBlock as u16), - _ if child.kind() == "arguments" => child - .children() - .filter(|c| c.kind_id() == E::Keywords as u16) - .flat_map(|keywords| keywords.children()) - .filter_map(|pair| pair.child_by_field_name("key")) - .any(|key| key.utf8_text(code).is_some_and(|k| k.trim_end() == "else:")), + _ if child.kind() == "arguments" => elixir_keyword_section(&child, code, "else").is_some(), _ => false, }) } +// The operands of a short-circuit `Kernel` remote call +// (`elixir_applying_remote_call`), each a boolean slot as the `left` and +// `right` of `a || b` are, so `Kernel.||(a, b)` scores what `a || b` +// does (#1574): the call's arguments, and under a pipe +// (`a |> Kernel.||(b)`) the piped value, which the pipe passes as the +// first argument. `up` yields the ancestors above the call. +// +// Which side of the pipe the call sits on is not checked: in +// `Kernel.||(a, b) |> f()` the piped value is the call itself, which +// `elixir_condition_scores_itself` already says pays nothing. +fn elixir_count_remote_chain<'a>( + call: &Node<'a>, + code: &[u8], + mut up: impl Iterator>, + conditions: &mut f64, +) { + let score = |operand: &Node, conditions: &mut f64| { + elixir_count_condition(operand, code, conditions); + }; + if let Some(piped) = up + .next() + .filter(|pipe| elixir_binary_operator_is(pipe, |op| op == Elixir::PIPEGT)) + .and_then(|pipe| pipe.child_by_field_name("left")) + { + score(&piped, conditions); + } + if let Some(arguments) = call.children().find(|child| child.kind() == "arguments") { + count_each_operand(&arguments, score, conditions); + } +} + // What an Elixir `Call` contributes. The classification is by keyword // text rather than by kind, so it is a paragraph of policy rather than a // dispatch arm — the same split `java_count_token_branch` and @@ -226,7 +261,7 @@ fn elixir_count_call(node: &Node, code: &[u8], stats: &mut Stats) { .find(|c| c.kind() == "arguments") .and_then(|args| wrapped_operand(&args)) { - elixir_count_condition(&predicate, &mut stats.conditions); + elixir_count_condition(&predicate, code, &mut stats.conditions); } // Fitzpatrick Rule 5 counts the `else`, as Ruby's, Java's, // Python's and Go's `if … else` each pay it (#1531). Only an @@ -367,7 +402,7 @@ impl Abc for ElixirCode { // and `Cyclomatic` impls share. E::When if elixir_when_is_guard(node, code, ancestors) => { if let Some(operator) = ancestors.parent(node) { - elixir_count_guard(&operator, &mut stats.conditions); + elixir_count_guard(&operator, code, &mut stats.conditions); } } // `in` / `not in` are Elixir's membership and type tests @@ -445,14 +480,27 @@ impl Abc for ElixirCode { // token's parent can be an `ERROR` node, which has no // operands to score. `a && b || c` is a left-nested chain of // `binary_operator`s, so an operand that is itself a chain is - // paid by its own operator's visit. + // paid by its own operator's visit. A short-circuit `Kernel` + // remote call applies the operator too, as `Cyclomatic` + // counts it through the same predicate (§7, #1574). E::AMPAMP | E::PIPEPIPE | E::And | E::Or => { if let Some(chain) = elixir_applying_operator(node, ancestors) { - count_field_operands(&chain, elixir_count_condition, &mut stats.conditions); + count_field_operands( + &chain, + |operand, conditions| elixir_count_condition(operand, code, conditions), + &mut stats.conditions, + ); + } else if let Some(call) = elixir_applying_remote_call(node, code, ancestors) { + let above_call = ancestors + .iter(node) + .map(|(ancestor, _)| ancestor) + .skip_while(|ancestor| ancestor.id() != call.id()) + .skip(1); + elixir_count_remote_chain(&call, code, above_call, &mut stats.conditions); } } E::StabClause if elixir_clause_is_decision(node, code, ancestors) => { - elixir_count_clause(node, &mut stats.conditions); + elixir_count_clause(node, code, &mut stats.conditions); } // A `with`'s `pattern <- expr` clause is a pattern-match // condition (Rule 9), the decision `Cyclomatic` counts diff --git a/src/metrics/cognitive/elixir.rs b/src/metrics/cognitive/elixir.rs index b5facbd8..9a395f37 100644 --- a/src/metrics/cognitive/elixir.rs +++ b/src/metrics/cognitive/elixir.rs @@ -12,6 +12,45 @@ )] use super::*; +use crate::lang_helpers::elixir::elixir_applying_remote_call; +use crate::metrics::elixir_decisions::elixir_keyword_pair_section; + +/// Whether a Call spelling `keyword` takes `try`'s sections — `else`, +/// `rescue`, `catch` — in its keyword form: `try` itself, and the +/// method-defining macros, whose body is an implicit `try` +/// (`def f, do: g(), rescue: (e -> e)`). +fn elixir_takes_try_sections(keyword: &str) -> bool { + keyword == "try" || elixir_is_method_macro(keyword) +} + +/// The cost a keyword-form section carries, matching its block form. +enum ElixirSection { + /// `else:`, which adds +1 as the `Else` token does. + Else, + /// `rescue:` / `catch:`, which add +nesting as `RescueBlock` / + /// `CatchBlock` do. + Handler, +} + +/// The section `pair` spells, when its Call takes it (#1574). Only such +/// a Call scores it, because `else:` is an ordinary option name to any +/// other function; a `with`'s `else:` is that construct's clause +/// dispatch, which its own +nesting already paid. +fn elixir_section<'a>( + pair: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> Option { + let (key, call) = elixir_keyword_pair_section(pair, code, ancestors)?; + let keyword = elixir_call_keyword(&call, code)?; + match key { + "else" if matches!(keyword, "if" | "unless") || elixir_takes_try_sections(keyword) => { + Some(ElixirSection::Else) + } + "rescue" | "catch" if elixir_takes_try_sections(keyword) => Some(ElixirSection::Handler), + _ => None, + } +} /// Folds an Elixir `BinaryOperator`'s short-circuit operator children /// into the boolean-sequence counter — Elixir has four (`&&`, `||`, @@ -26,6 +65,24 @@ fn compute_elixir_booleans(node: &Node, stats: &mut Stats) { }); } +/// [`compute_elixir_booleans`] for a short-circuit `Kernel` remote call +/// (`Kernel.||(a, b)`), whose operator token sits in the call's name +/// rather than among a `binary_operator`'s children. The sequence is +/// keyed on the call's span, as a `binary_operator`'s is on its own. +fn compute_elixir_remote_booleans<'a>( + op: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, + stats: &mut Stats, +) { + if let Some(call) = elixir_applying_remote_call(op, code, ancestors) { + stats.structural = + stats + .boolean_seq + .eval_based_on_prev(op.kind_id(), call.end_byte(), stats.structural); + } +} + impl Cognitive for ElixirCode { // Elixir control flow is macro-shaped: `if`, `unless`, `case`, // `cond`, `with`, `for`, `while`, and `try` each surface as a @@ -51,7 +108,8 @@ impl Cognitive for ElixirCode { // `FunctionDefinition` rule, but the function depth deliberately // does NOT bump — see the arm comment below for why. // - `AnonymousFunction` (`fn x -> y end`): lambda nesting bumps. - // - `&&` / `||` / `and` / `or`: boolean sequence cost. + // - `&&` / `||` / `and` / `or`: boolean sequence cost, as is a + // `Kernel.&&` / `Kernel.||` remote call. // // Limitations: // - `Enum.reduce` / `Enum.map` and friends are higher-order function @@ -66,10 +124,11 @@ impl Cognitive for ElixirCode { fn compute<'a>( node: &Node<'a>, code: &'a [u8], - _ancestors: Ancestors<'a, '_>, + ancestors: Ancestors<'a, '_>, stats: &mut Stats, nesting_map: &mut NestingMap, ) { + // bca: suppress(cyclomatic, halstead) — kind dispatch table; each gated arm is one rule use Elixir as E; let mut nesting = get_nesting_from_map(node, nesting_map); @@ -117,11 +176,36 @@ impl Cognitive for ElixirCode { _ => {} }, // `else` keyword inside an `else_block` (else arm of `if` - // / `unless` / `with` / `try`). Matches Java/Kotlin's - // `Else` rule: +1 without raising nesting. - E::Else => { + // / `unless` / `try`). Matches Java/Kotlin's `Else` rule: +1 + // without raising nesting. + // + // Not a `with`'s: its `else` holds the clauses that dispatch + // over the value a `<-` failed to match, the arms of the + // `case` the construct is in all but spelling, and `case`'s + // arms add nothing past the +nesting its container paid. + // Ruby draws the same line for the `else` of `case/when` + // (#451); the `if` alternative and `try`'s no-exception + // branch, which no container paid for, keep their +1. The + // keyword form `with(…, do: …, else: (…))` scored 1 all + // along; this puts the block form level with it (#1574). + // The `else` token's parents are the `else_block`, then the + // `do_block`, then the Call. + E::Else + if ancestors + .iter(node) + .nth(2) + .is_none_or(|(call, _)| elixir_call_keyword(&call, code) != Some("with")) => + { increment_by_one(stats); } + // The keyword spellings of the sections above: + // `if(c, do: 1, else: 2)`, `try(do: f(), rescue: (…))`, bare + // or in brackets. + E::Pair => match elixir_section(node, code, ancestors) { + Some(ElixirSection::Else) => increment_by_one(stats), + Some(ElixirSection::Handler) => increase_nesting(stats, &mut nesting), + None => {} + }, // `rescue` / `catch` arms of a `try` Call each add +nesting, // matching Java's `CatchClause` treatment. E::RescueBlock | E::CatchBlock => { @@ -143,6 +227,15 @@ impl Cognitive for ElixirCode { E::BinaryOperator | E::BinaryOperator2 | E::BinaryOperator3 => { compute_elixir_booleans(node, stats); } + // A short-circuit `Kernel` remote call applies the operator + // as `a || b` does, so it opens or continues a sequence the + // same way (#1574), keyed on the call's span as a + // `binary_operator`'s operator is keyed on the operator's. + // `Foo.||(a, b)` is an ordinary call and is not matched; the + // token inside a `binary_operator` is the arm above's. + E::AMPAMP | E::PIPEPIPE => { + compute_elixir_remote_booleans(node, code, ancestors, stats); + } _ => {} } nesting_map.insert(node.id(), nesting); diff --git a/src/metrics/cyclomatic/elixir.rs b/src/metrics/cyclomatic/elixir.rs index 30793877..4cc0142b 100644 --- a/src/metrics/cyclomatic/elixir.rs +++ b/src/metrics/cyclomatic/elixir.rs @@ -7,6 +7,7 @@ )] use super::*; +use crate::lang_helpers::elixir::{elixir_applying_operator, elixir_applying_remote_call}; use crate::metrics::elixir_decisions::{ elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_when_is_guard, elixir_with_else_dispatches, @@ -121,14 +122,11 @@ impl Cyclomatic for ElixirCode { // on the same parent. A remote call naming it, // `Kernel.||(a, b)`, applies the macro and decides like // `a || b`; a user module's `Foo.||(a, b)` is an ordinary - // call and does not. ABC and cognitive still score the - // `Kernel` call 0, as they did before #1534. + // call and does not. ABC and cognitive score the `Kernel` + // call through the same predicate (#1574). E::AMPAMP | E::PIPEPIPE | E::And | E::Or - if crate::lang_helpers::elixir::elixir_applying_operator(node, ancestors) - .is_some() - || crate::lang_helpers::elixir::elixir_remote_call_applies( - node, code, ancestors, - ) => + if elixir_applying_operator(node, ancestors).is_some() + || elixir_applying_remote_call(node, code, ancestors).is_some() => { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; diff --git a/src/metrics/elixir_decisions.rs b/src/metrics/elixir_decisions.rs index 03f891fa..d1a425bd 100644 --- a/src/metrics/elixir_decisions.rs +++ b/src/metrics/elixir_decisions.rs @@ -8,6 +8,7 @@ //! one module. use crate::lang_helpers::elixir::{elixir_call_keyword, elixir_is_method_macro}; +use crate::metrics::abc::is_operand; use crate::{Ancestors, Elixir, Node}; /// Whether `node` is a `binary_operator` applying `operator`. @@ -172,8 +173,6 @@ fn elixir_is_guard_position<'a>( ) -> bool { use Elixir as E; - const ARGUMENTS: &str = "arguments"; - if parent.kind_id() == E::StabClause as u16 { return parent .child_by_field_name("left") @@ -373,9 +372,9 @@ pub(super) fn elixir_clause_is_decision<'a>( | E::CatchBlock | E::AnonymousFunction ) || (parent.kind_id() == E::Block as u16 - && up - .next() - .is_some_and(|pair| elixir_is_section_pair(&pair, code, SECTION_KEYS))) + && up.next().is_some_and(|pair| { + elixir_pair_key(&pair, code).is_some_and(|key| SECTION_KEYS.contains(&key)) + })) }); if !in_section { return false; @@ -397,8 +396,9 @@ pub(super) fn elixir_clause_is_decision<'a>( /// the failed value, which collapses into one decision exactly when it /// holds one — so `else _ -> 0`, which picks nothing, stays free in /// modified as it is in standard, like Rust's `if let … else` (#1535). -/// Spelled in either form: the block form's `else_block`, or the -/// keyword form's `else:` pair whose value is a parenthesised `block`. +/// Spelled in any form: the block form's `else_block`, or the keyword +/// form's `else:` pair whose value is a parenthesised `block`, bare or +/// bracketed ([`elixir_trailing_keywords`]). /// /// In a `with`'s `else` the default-clause rule reduces to the bare /// `_`: `true ->` is free only under a `cond`, and the head skip only @@ -412,12 +412,7 @@ pub(super) fn elixir_with_else_dispatches(call: &Node, code: &[u8]) -> bool { E::DoBlock => child .children() .find(|c| c.kind_id() == E::ElseBlock as u16), - _ if child.kind() == "arguments" => child - .children() - .filter(|c| c.kind_id() == E::Keywords as u16) - .flat_map(|keywords| keywords.children()) - .find(|pair| elixir_is_section_pair(pair, code, &["else:"])) - .and_then(|pair| pair.child_by_field_name("value")), + _ if child.kind() == ARGUMENTS => elixir_keyword_section(&child, code, "else"), _ => None, }; section.is_some_and(|section| { @@ -433,23 +428,125 @@ pub(super) fn elixir_with_else_dispatches(call: &Node, code: &[u8]) -> bool { /// The keyword spellings of the clause sections a `do … end` block /// spells as `do_block` / `else_block` / `after_block` / `rescue_block` /// / `catch_block`. -const SECTION_KEYS: &[&str] = &["do:", "else:", "after:", "rescue:", "catch:"]; +const SECTION_KEYS: &[&str] = &["do", "else", "after", "rescue", "catch"]; + +/// The key a keyword-list `pair` spells, without its colon: `do` for +/// both `do: x` and the quoted `"do": x`, which Elixir reads as the same +/// atom. The key token's text carries its colon and the whitespace after +/// it. `None` for any other node, and for a quoted key that interpolates +/// (`"do#{k}": x`), whose atom is not known until run time — valid in an +/// ordinary call's options, but never a section. +/// +/// The grammar's key is a `keyword` token, which has no named parts, or a +/// `quoted_keyword` around its `quoted_content` and any interpolations. +fn elixir_pair_key<'a>(pair: &Node<'a>, code: &'a [u8]) -> Option<&'a str> { + let key = Some(pair) + .filter(|pair| pair.kind_id() == Elixir::Pair as u16) + .and_then(|pair| pair.child_by_field_name("key"))?; + let mut parts = key.children().filter(Node::is_named); + match (parts.next(), parts.next()) { + (None, _) => key + .utf8_text(code) + .and_then(|text| text.trim_end().strip_suffix(':')), + (Some(content), None) if content.kind() == "quoted_content" => content.utf8_text(code), + _ => None, + } +} + +/// The keyword list `arguments` ends with: a bare `keywords` node +/// (`if c, do: x`), or the same list in brackets (`if(c, [do: x])`), +/// which Elixir reads as the identical trailing keyword list — the +/// keyword syntax is sugar for exactly that last-argument literal. +/// +/// Only the *last* argument qualifies, and only a list holding nothing +/// but the `keywords` node: `[a: 1]` in any other position is an +/// ordinary list argument, and `[1, do: x]` is a list with a keyword +/// tail, not a keyword list. An empty `[]` holds no `keywords` at all. +/// The grammar ends a list with its `keywords`, so a list whose first +/// item is one holds nothing else. +/// +/// Matched by rule name: `keywords` carries two kind ids at this pin +/// (grammar-dispatch §1). Comments are skipped as the extras they are. +pub(super) fn elixir_trailing_keywords<'a>(arguments: &Node<'a>) -> Option> { + let last = arguments.children().filter(is_operand).last()?; + if last.kind() == KEYWORDS { + return Some(last); + } + if last.kind_id() != Elixir::List as u16 { + return None; + } + last.children() + .find(is_operand) + .filter(|keywords| keywords.kind() == KEYWORDS) +} + +/// The value of the `key:` pair in `arguments`' trailing keyword list +/// ([`elixir_trailing_keywords`]), or `None` when it has no such pair. +pub(super) fn elixir_keyword_section<'a>( + arguments: &Node<'a>, + code: &'a [u8], + key: &str, +) -> Option> { + elixir_trailing_keywords(arguments)? + .children() + .find(|pair| elixir_pair_key(pair, code) == Some(key))? + .child_by_field_name("value") +} -/// Whether `pair` is a keyword-form section (`do: (…)`) whose key is one -/// of `keys`. The key token's text carries its colon and the whitespace -/// after it, so it is trimmed before comparing. -fn elixir_is_section_pair(pair: &Node, code: &[u8], keys: &[&str]) -> bool { - pair.kind_id() == Elixir::Pair as u16 - && pair - .child_by_field_name("key") - .and_then(|key| key.utf8_text(code)) - .is_some_and(|key| keys.contains(&key.trim_end())) +/// The section key of `pair` and the `Call` whose trailing keyword list +/// holds it — `("else", if …)` for the `else:` of `if(c, do: 1, else: 2)` +/// or of `if(c, [do: 1, else: 2])` — or `None` when `pair` is not a +/// keyword argument of a Call. `up` yields the ancestors above `pair`. +/// +/// The climb is `keywords`, an optional bracketing `list`, `arguments`, +/// then the Call; the keyword list must be the one +/// [`elixir_trailing_keywords`] finds, so a `[do: x]` in any other +/// argument position, a map's keyword pairs and a nested list are not +/// sections. That identity check is also what rejects a list nested in +/// a list (`f([[do: x]])`), whose inner list the climb steps over. +fn elixir_pair_section<'a>( + pair: &Node<'a>, + code: &'a [u8], + mut up: impl Iterator>, +) -> Option<(&'a str, Node<'a>)> { + let key = elixir_pair_key(pair, code)?; + let keywords = up.next().filter(|keywords| keywords.kind() == KEYWORDS)?; + let arguments = up.find(|node| node.kind_id() != Elixir::List as u16)?; + up.next() + .filter(|call| { + call.kind_id() == Elixir::Call as u16 + && arguments.kind() == ARGUMENTS + && elixir_trailing_keywords(&arguments).map(|found| found.id()) + == Some(keywords.id()) + }) + .map(|call| (key, call)) } +/// [`elixir_pair_section`] for a `pair` the walker is visiting, its +/// ancestors read off the walker's chain. +pub(super) fn elixir_keyword_pair_section<'a>( + pair: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> Option<(&'a str, Node<'a>)> { + elixir_pair_section( + pair, + code, + ancestors.iter(pair).map(|(ancestor, _)| ancestor), + ) +} + +/// The rule name of a keyword list (two kind ids at this pin). +const KEYWORDS: &str = "keywords"; + +/// The rule name of an argument list (five kind ids at this pin). +const ARGUMENTS: &str = "arguments"; + /// The `Call` whose `do` section holds `node` (a `stab_clause`): the /// block form's `do_block` (whose parent is the call) or the keyword -/// form's `do: (…)` block (`block` → `pair` → `keywords` → `arguments` -/// → call). The caller asks the Call's keyword (`cond`, `for`). +/// form's `do: (…)` block, bare or bracketed (`block` → `pair` → the +/// climb [`elixir_pair_section`] makes). The caller asks the Call's +/// keyword (`cond`, `for`). fn elixir_do_section_call<'a>( node: &Node<'a>, code: &'a [u8], @@ -460,10 +557,12 @@ fn elixir_do_section_call<'a>( let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); match up.next().map(|parent| parent.kind_id().into()) { Some(E::DoBlock) => up.next(), - Some(E::Block) => up - .next() - .filter(|pair| elixir_is_section_pair(pair, code, &["do:"])) - .and_then(|_| up.nth(2)), + Some(E::Block) => { + let pair = up.next()?; + elixir_pair_section(&pair, code, up) + .filter(|(key, _)| *key == "do") + .map(|(_, call)| call) + } _ => None, } } diff --git a/src/metrics/elixir_spelling_parity_tests.rs b/src/metrics/elixir_spelling_parity_tests.rs new file mode 100644 index 00000000..9fff50e0 --- /dev/null +++ b/src/metrics/elixir_spelling_parity_tests.rs @@ -0,0 +1,344 @@ +//! Two spellings of one Elixir program score alike (#1574). +//! +//! Elixir writes a construct's sections three ways: a `do … end` block, a +//! trailing keyword list (`if c, do: 1, else: 2`), and the same list in +//! brackets (`if(c, [do: 1, else: 2])`), which the compiler reads as the +//! identical last argument. And it applies a short-circuit operator either +//! as an operator (`a || b`) or as a `Kernel` remote call +//! (`Kernel.||(a, b)`), which expands the same macro. +//! +//! Every group pairs the spellings with a twin whose score predates the +//! fix — the block form, or the operator form — and that is the oracle: +//! the twin's numbers are asserted outright, so a group cannot pass by +//! every spelling drifting together. The one twin that moved is the +//! `with … else` block form's cognitive score, decided in +//! [`elixir_with_else_scores_like_case`] against `case`. + +use std::cell::Cell; + +use crate::test_support::check_func_space_only; +use crate::*; + +/// `[cyclomatic, cyclomatic_modified, abc conditions, cognitive]` sums of +/// the root space; both cyclomatic tiers carry a base of 1. +#[cfg(feature = "elixir")] +type Scores = [u64; 4]; + +#[cfg(feature = "elixir")] +fn measure(source: &str) -> Scores { + let out = Cell::new([0; 4]); + check_func_space_only::( + &format!("{source}\n"), + "foo.exs", + &[Metric::Cyclomatic, Metric::Abc, Metric::Cognitive], + |space| { + let m = &space.metrics; + out.set([ + m.cyclomatic.cyclomatic_sum(), + m.cyclomatic.cyclomatic_modified_sum(), + m.abc.conditions_sum(), + m.cognitive.cognitive_sum(), + ]); + }, + ); + out.get() +} + +/// Asserts `twin` scores `want`, then that every spelling scores the same. +#[cfg(feature = "elixir")] +fn assert_group(twin: &str, want: Scores, spellings: &[&str]) { + assert_eq!( + measure(twin), + want, + "twin `{twin}` moved; re-derive the group" + ); + for spelling in spellings { + assert_eq!( + measure(spelling), + want, + "`{spelling}` must score like `{twin}`" + ); + } +} + +/// A `Kernel` remote call naming `&&` / `||` is the operator: one +/// decision, one ABC slot per operand, one cognitive sequence. Before +/// #1574 only cyclomatic counted it; ABC and cognitive scored 0, and a +/// nested call read as a plain operand, so `a || Kernel.||(b, c)` paid a +/// slot for the inner call that `a || (b || c)` does not. +#[cfg(feature = "elixir")] +#[test] +fn elixir_kernel_remote_call_scores_like_the_operator() { + assert_group( + "a || b", + [2, 2, 2, 1], + &[ + "Kernel.||(a, b)", + "Elixir.Kernel.||(a, b)", + // The pipe passes the piped value as the first operand. + "a |> Kernel.||(b)", + ], + ); + assert_group("a && b", [2, 2, 2, 1], &["Kernel.&&(a, b)"]); + // A nested chain pays its own operands and no slot as an operand. + assert_group( + "a || (b || c)", + [3, 3, 3, 1], + &[ + "Kernel.||(a, Kernel.||(b, c))", + "a || Kernel.||(b, c)", + "Kernel.||(a, b || c)", + ], + ); + // A change of operator opens a second cognitive sequence. + assert_group( + "a && (b || c)", + [3, 3, 3, 2], + &["Kernel.&&(a, Kernel.||(b, c))"], + ); + // Piping the call's *result* passes no operand to it. + assert_group("(a || b) |> f()", [2, 2, 2, 1], &["Kernel.||(a, b) |> f()"]); + // As an `if` predicate the call scores itself, as the chain does. + assert_group( + "if a || b, do: 1", + [3, 3, 2, 2], + &["if Kernel.||(a, b), do: 1"], + ); +} + +/// A user module's `||/2` is an ordinary function, and a capture only +/// names `Kernel`'s: neither decides anything in any metric. +#[cfg(feature = "elixir")] +#[test] +fn elixir_non_kernel_remote_call_decides_nothing() { + assert_group( + "Foo.foo(a, b)", + [1, 1, 0, 0], + &["Foo.||(a, b)", "a |> Foo.||(b)", "&Kernel.||/2"], + ); + // In a slot, neither a user module's `||` nor a `Kernel` function + // that is no operator scores itself: each pays the slot as a call. + assert_group( + "if foo(a), do: 1", + [2, 2, 1, 1], + &["if Kernel.foo(a), do: 1", "if Foo.||(a, b), do: 1"], + ); +} + +/// `if` / `unless` with an `else`, in every spelling of the keyword list, +/// including a trailing comma and quoted keys (`"do":` is the atom +/// `:do`). Before #1574 the keyword forms lost cognitive's `else` +1, +/// and the bracketed and quoted forms lost ABC's `else` condition too. +#[cfg(feature = "elixir")] +#[test] +fn elixir_if_else_scores_alike_in_every_spelling() { + assert_group( + "if c do 1 else 2 end", + [2, 2, 2, 2], + &[ + "if(c, do: 1, else: 2)", + "if(c, [do: 1, else: 2])", + "if(c, [do: 1, else: 2,])", + "if(c, \"do\": 1, \"else\": 2)", + "if(c, [\"do\": 1, \"else\": 2])", + ], + ); + assert_group( + "unless c do 1 else 2 end", + [2, 2, 2, 2], + &["unless(c, do: 1, else: 2)", "unless(c, [do: 1, else: 2])"], + ); + // Each `else` belongs to its own `if`, however the two nest. + assert_group( + "if a do (if b do 1 else 2 end) else 3 end", + [3, 3, 4, 5], + &[ + "if(a, do: if(b, do: 1, else: 2), else: 3)", + "if(a, [do: if(b, [do: 1, else: 2]), else: 3])", + ], + ); +} + +/// Only the *last* argument, and only a list holding nothing but pairs, +/// is a keyword list: `[do: 1, else: 2]` in first position, `[1, do: …]` +/// and `[]` are ordinary list arguments, so the `if` has no `else`. And +/// `else:` is an ordinary option name to any other function. +#[cfg(feature = "elixir")] +#[test] +fn elixir_list_that_is_not_a_keyword_list_has_no_sections() { + assert_group( + "if(c, x)", + [2, 2, 1, 1], + &[ + "if([do: 1, else: 2], c)", + "if(c, [1, do: 1, else: 2])", + "if(c, [])", + ], + ); + assert_group( + "f(x, y)", + [1, 1, 0, 0], + &["f(x, else: 1)", "f(x, [else: 1])"], + ); + assert_group( + "Foo.bar(x, y)", + [1, 1, 0, 0], + &["Foo.bar(x, else: 1)", "Foo.bar(x, [else: 1])"], + ); + // An interpolated key has no static name. Only an ordinary call + // takes one validly, and no ordinary call has sections, so this row + // exercises the reading rather than discriminating it. + assert_group("f(x, y)", [1, 1, 0, 0], &["f(x, \"else#{y}\": 1)"]); +} + +/// Clause constructs: the bracketed `cond` lost its free `true ->` +/// default, because the `cond` lookup climbed a fixed number of parents. +#[cfg(feature = "elixir")] +#[test] +fn elixir_clause_constructs_score_alike_in_every_spelling() { + assert_group( + "case x do 1 -> :a; 2 -> :b end", + [3, 2, 2, 1], + &[ + "case(x, do: (1 -> :a; 2 -> :b))", + "case(x, [do: (1 -> :a; 2 -> :b)])", + ], + ); + assert_group( + "cond do x > 1 -> :a; true -> :b end", + [2, 2, 1, 1], + &[ + "cond(do: (x > 1 -> :a; true -> :b))", + "cond([do: (x > 1 -> :a; true -> :b)])", + ], + ); +} + +/// `try`'s `rescue:` / `catch:` pay +nesting and its `else:` +1 in the +/// keyword forms, as their blocks do; before #1574 the keyword `try` +/// scored cognitive 0. A `def` body is an implicit `try` and takes the +/// same sections. +#[cfg(feature = "elixir")] +#[test] +fn elixir_try_sections_score_alike_in_every_spelling() { + assert_group( + "try do f() rescue e in RuntimeError -> e catch :x -> 1 end", + [3, 2, 2, 2], + &[ + "try(do: f(), rescue: (e in RuntimeError -> e), catch: (:x -> 1))", + "try([do: f(), rescue: (e in RuntimeError -> e), catch: (:x -> 1)])", + ], + ); + assert_group( + "try do f() rescue _ -> 1 else x -> x end", + [2, 2, 1, 2], + &[ + "try(do: f(), rescue: (_ -> 1), else: (x -> x))", + "try([do: f(), rescue: (_ -> 1), else: (x -> x)])", + ], + ); + // A handler raises nesting, not just +1: the `if` inside it costs 2. + assert_group( + "try do f() rescue _ -> (if a do 1 end) end", + [2, 3, 1, 3], + &[ + "try(do: f(), rescue: (_ -> if(a, do: 1)))", + "try([do: f(), rescue: (_ -> if(a, do: 1))])", + ], + ); + assert_group( + "try do f() catch _ -> (if a do 1 end) end", + [2, 3, 1, 3], + &["try(do: f(), catch: (_ -> if(a, do: 1)))"], + ); + assert_group( + "def f() do\n g()\nrescue\n e -> e\nend", + [3, 2, 1, 1], + &["def f(), do: g(), rescue: (e -> e)"], + ); +} + +/// A `with`'s `else` holds the clauses that dispatch over the value a +/// `<-` failed to match: the arms of a `case` in all but spelling. A +/// `case`'s arms add no cognitive cost past the +nesting its container +/// pays, and Ruby draws the same line for the `else` of `case/when` +/// (#451), so the block form's `else` token adds nothing either; the +/// keyword form already scored that (#1574). An `if`'s `else` is a +/// conditional's alternative and keeps its +1 (see the nested row). +/// +/// The bracketed form also lost its modified tier: the `else` lookup +/// read only a bare keyword list, so the dispatching `else` looked absent. +#[cfg(feature = "elixir")] +#[test] +fn elixir_with_else_scores_like_case() { + // The oracle: the same dispatch as one `case`, cognitive 1. + assert_eq!( + measure("case g(x) do {:ok, a} -> a; {:error, e} -> e; :z -> 0 end")[3], + 1 + ); + assert_group( + "with({:ok, a} <- g(x), do: a, else: ({:error, e} -> e; :z -> 0))", + [4, 3, 3, 1], + &[ + "with {:ok, a} <- g(x) do a else {:error, e} -> e; :z -> 0 end", + "with({:ok, a} <- g(x), [do: a, else: ({:error, e} -> e; :z -> 0)])", + ], + ); + // A non-dispatching `else _ -> 0` is free in the modified tier too. + assert_group( + "with({:ok, a} <- g(x), do: a, else: (_ -> 0))", + [2, 2, 1, 1], + &[ + "with {:ok, a} <- g(x) do a else _ -> 0 end", + "with({:ok, a} <- g(x), [do: a, else: (_ -> 0)])", + ], + ); + // The `if` nested in the body keeps its own `else` +1 at nesting 1. + assert_group( + "with({:ok, a} <- g(x), do: if(a, do: 1, else: 2), else: (_ -> 0))", + [3, 3, 3, 4], + &["with {:ok, a} <- g(x) do (if a do 1 else 2 end) else _ -> 0 end"], + ); +} + +/// A `for … reduce:`'s first accumulator clause is its head, free in +/// every spelling (#1535). The bracketed form missed the `for` behind +/// the list and scored the head as a decision. +#[cfg(feature = "elixir")] +#[test] +fn elixir_for_reduce_head_is_free_in_every_spelling() { + assert_group( + "for x <- xs, reduce: 0 do acc -> acc + x end", + [2, 2, 0, 1], + &[ + "for(x <- xs, reduce: 0, do: (acc -> acc + x))", + "for(x <- xs, [reduce: 0, do: (acc -> acc + x)])", + ], + ); + assert_group( + "for x <- xs, reduce: 0 do 0 -> x; acc -> acc + x end", + [3, 2, 1, 1], + &[ + "for(x <- xs, reduce: 0, do: (0 -> x; acc -> acc + x))", + "for(x <- xs, [reduce: 0, do: (0 -> x; acc -> acc + x)])", + ], + ); +} + +/// `receive` agrees across spellings, but its modified and cognitive +/// scores are wrong in all of them, so the group asserts only agreement. +/// That agreement predates #1574 — its clauses score through the +/// owner-blind section gate (#1583), not the trailing-list lookup — so +/// this guards the spellings drifting apart, not that fix. +// FIXME(#1581): pin the values once `receive` pays like `case`. +#[cfg(feature = "elixir")] +#[test] +fn elixir_receive_scores_alike_in_every_spelling() { + let block = measure("receive do {:a, x} -> x; :b -> 0 after 10 -> :t end"); + for spelling in [ + "receive(do: ({:a, x} -> x; :b -> 0), after: (10 -> :t))", + "receive([do: ({:a, x} -> x; :b -> 0), after: (10 -> :t)])", + ] { + assert_eq!(measure(spelling), block, "`{spelling}`"); + } +} diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 1d76d550..6ed5f3ad 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -78,6 +78,9 @@ mod cpp_alternative_tokens_tests; #[path = "elixir_named_operator_tests.rs"] mod elixir_named_operator_tests; #[cfg(test)] +#[path = "elixir_spelling_parity_tests.rs"] +mod elixir_spelling_parity_tests; +#[cfg(test)] #[path = "perl_keyword_key_tests.rs"] mod perl_keyword_key_tests; From 9e4fc2e88368ac1ec8b0ec68ea99a4e4b904e2b0 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:01:01 -0700 Subject: [PATCH 15/34] test(bench): add Perl and Bash scaling probes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven probes for the walks #1273, #1536 and #1540 made quadratic, which the complexity-class gate could not see: a Perl `||` chain and a chain a misparsed comparison absorbed (the spine climb), a block of `&&` statements (the stop parent's operator lookup), a file whose tail recovers into a flat ERROR root (perl_binding_rewrites), a Bash `[ … -o … ]` run (cognitive), a valid `(( a = … = 0 ))` and a `[[ … = … && … ]]` run (ABC). Each fails at 1.91-2.06 with its pre-fix walk restored and holds at 0.98-1.19 clean. The ERROR-tail probe is the first whose shape must not parse; RECOVERY_PROBES exempts it from the parse check and pins its ERROR root. Its body carries only `=~`: a bareword or chain under a flat ERROR is still quadratic today (#1584). Fixes #1575 --- big-code-analysis-bench/src/shapes.rs | 295 +++++++++++++++++++++++++- docs/development/benchmarking.md | 43 +++- 2 files changed, 333 insertions(+), 5 deletions(-) diff --git a/big-code-analysis-bench/src/shapes.rs b/big-code-analysis-bench/src/shapes.rs index 1926f1ba..9abfa256 100644 --- a/big-code-analysis-bench/src/shapes.rs +++ b/big-code-analysis-bench/src/shapes.rs @@ -557,6 +557,122 @@ pub fn kotlin_noinline_params(width: usize) -> String { source } +/// Perl: `my $r = $v || $v || … || $v;`. +/// +/// `||` is left-associative, so the chain is a left-nested spine one +/// `binary_expression` deeper per operand. Perl ABC asks every chain +/// node whether a misparsed comparison (#1273) absorbed its leading +/// operand, by climbing the spine of leading slots to its root. Every +/// node but the bottom one leads with a chain, which scores itself, so +/// only the bottom one climbs; climbing from each was `O(depth)` per +/// operator: ~15 s for 8 000 operands in a release build. +#[must_use] +pub fn perl_or_chain(depth: usize) -> String { + format!("my $r = {}$v;\n", "$v || ".repeat(depth)) +} + +/// Perl: `my $r = $x > $v && $v && … && $v;`. +/// +/// [`perl_or_chain`] behind the #1273 misparse it exists to detect: +/// tree-sitter-perl lets `>` swallow the chain to its right, so the +/// whole spine is the comparison's right operand and the bottom node's +/// climb ends on the `>`, answering "absorbed". The two shapes share +/// the climb and differ in where it stops. +#[must_use] +pub fn perl_misparsed_comparison(depth: usize) -> String { + format!("my $r = $x > {}$v;\n", "$v && ".repeat(depth)) +} + +/// Perl: `sub f { $a && $b; $a && $b; … }` — one block of `width` +/// two-operand chains. +/// +/// The width shape for the same misparse check. Each chain's leading +/// operand is a bare variable, so each climbs, and the climb stops one +/// step up at the block to read the operator before the chain. +/// Scanning the block's statements from its start for it was +/// `O(width)` per chain: 16.3 s for 10 000 statements in a release +/// build. +#[must_use] +pub fn perl_and_statements(width: usize) -> String { + format!("sub f {{ {}}}\n", "$a && $b; ".repeat(width)) +} + +/// Perl: `my @r = (` followed by `width` copies of `$d $c =~ /x/;⏎{⏎`, +/// never closed. +/// +/// The width shape for a file whose tail fails to parse, as +/// `perl5db.pl`'s does: tree-sitter-perl gives up on the unclosed list, +/// the root becomes an `ERROR`, and every `=~` in the tail is its direct +/// child. Perl ABC asks each `=~` whether it binds a rewrite, which +/// reads its parent's last operand; read off the `ERROR`, that is a +/// scan of the whole tail per `=~`. The body is the smallest found that +/// keeps the `=~` flat: without the stray `$d` and `{` it folds back +/// into a narrow `binary_expression`. +/// +/// FIXME(#1584): a bareword or a `&&` chain in this body makes the probe +/// quadratic on today's tree, because a flat `ERROR` defeats the +/// `O(log n)` child seek `Node::children_after` / `children_before` +/// rely on. Add `last if` and `$a && $b,` to the body with that fix. +#[must_use] +pub fn perl_error_tail(width: usize) -> String { + format!("my @r = ({}", "$d $c =~ /x/;\n{\n".repeat(width)) +} + +/// Bash: `if [ "$a" = 0 -o "$a" = 0 -a … ]; then :; fi`, the +/// connectives alternating. +/// +/// tree-sitter-bash lets `=` swallow the connectives around it, so the +/// test parses as one left-nested spine of comparisons, one level per +/// term (#1536). Bash cognitive ends each connective's boolean +/// sequence where the outermost comparison of its run does; finding +/// that end by climbing the run from every `binary_expression` was +/// `O(depth)` per node: 1.9 s for 4 000 terms in a release build. The +/// connectives alternate so the reading — one sequence per switch, the +/// textual-order scoring the mis-nested tree forces — grows with the +/// depth. +#[must_use] +pub fn bash_test_connective_run(depth: usize) -> String { + let mut source = String::from("if [ "); + for i in 0..depth { + source.push_str(if i % 2 == 0 { + "\"$a\" = 0 -o " + } else { + "\"$a\" = 0 -a " + }); + } + source.push_str("\"$a\" = 0 ]; then :; fi\n"); + source +} + +/// Bash: `(( a = a = … = 0 ))`. +/// +/// A valid chained arithmetic assignment, one `binary_expression` +/// deeper per `=`. The same `=` token is a string comparison inside +/// `[ … ]` / `[[ … ]]`, and Bash ABC once told the two apart by +/// climbing from every `=` through the expression nodes above it to +/// see whether a test held them — `O(depth)` per `=` here: 3.4 s for +/// 8 000 in a release build. Each test now counts its own comparisons +/// from the top, so an arithmetic `=` asks nothing. +#[must_use] +pub fn bash_arith_assign_chain(depth: usize) -> String { + format!("(( {}0 ))\n", "a = ".repeat(depth)) +} + +/// Bash: `if [[ $a = 0 && $a = 0 && … ]]; then :; fi`. +/// +/// The `[[ … ]]` form of the [`bash_test_connective_run`] misparse: +/// `=` swallows the `&&` around it, so the test is one deep spine. It +/// is the shape the comparison-counting change in Bash ABC has to stay +/// linear on, since every `=` in it is a comparison: 1.2 s for 16 000 +/// terms in a release build while each `=` climbed to its test. +#[must_use] +pub fn bash_eq_and_test(depth: usize) -> String { + format!( + "if [[ {}$a = 0 ]]; then :; fi\n", + "$a = 0 && ".repeat(depth) + ) +} + /// C++: `int f(int a){ return a + a + … + a; }`. /// /// The shape control for [`cpp_and_chain`]: the same left-nested @@ -1421,6 +1537,93 @@ pub const PROBES: &[Probe] = &[ pattern. The reading is one condition per bare \ match, so it grows with the width.", }, + Probe { + name: "abc/perl-or-chain", + lang: LANG::Perl, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc], + reading: |m| m.abc.conditions_sum(), + }, + render: perl_or_chain, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1273's misparse check climbs a chain's spine of \ + leading slots to ask whether a comparison absorbed \ + its first operand. A node whose leading slot scores \ + itself answers at once, so only the bottom of a \ + left-nested `||` spine climbs. Removing that early \ + answer, as before the fix, takes this probe from 1.00 \ + to 2.00 (8.8 ms to 2.33 s at depth 4 000) and \ + `abc/perl-misparsed-comparison` to 2.01, while every \ + other probe holds. The reading is one condition per \ + operand.", + }, + Probe { + name: "abc/perl-misparsed-comparison", + lang: LANG::Perl, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc], + reading: |m| m.abc.conditions_sum(), + }, + render: perl_misparsed_comparison, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "The same climb as `abc/perl-or-chain`, on a chain \ + the #1273 misparse really absorbed, so the bottom \ + node's climb ends on the `>` and answers yes. The two \ + probes fail together when the climb runs from every \ + node. The reading is one condition per operand.", + }, + Probe { + name: "abc/perl-and-statements", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc], + reading: |m| m.abc.conditions_sum(), + }, + render: perl_and_statements, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "The misparse check's climb from a two-operand \ + chain stops one step up, at the block, and asks for \ + the operator just before the chain. Scanning the \ + block from its start for it, as before the fix, takes \ + this probe from 1.08 to 2.01 (3.7 ms to 306 ms at \ + 2 000 statements). The kind gate that now skips a \ + block is not what this probe sees: with the gate alone \ + removed the lookup seeks the child by byte through \ + the block's balanced repeat and the probe holds at \ + 1.19. The reading is two conditions per statement.", + }, + Probe { + name: "abc/perl-error-tail", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc], + reading: |m| m.abc.conditions_sum(), + }, + render: perl_error_tail, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "`perl_binding_rewrites` asks each `=~` whether it \ + binds an `s///` / `tr///` by reading its parent's last \ + operand. In a file whose tail fails to parse, the \ + parent is an `ERROR` holding the whole tail, as in \ + `perl5db.pl`. Dropping the check that only a \ + `binary_expression` binds takes this probe from 1.19 \ + to 2.05 (2.9 ms to 492 ms at width 2 000). The only \ + probe whose shape does not parse (`RECOVERY_PROBES`). \ + The reading is two conditions per `=~` line, the \ + match and its pattern.", + }, Probe { name: "halstead/kotlin-noinline-params", lang: LANG::Kotlin, @@ -1548,6 +1751,68 @@ pub const PROBES: &[Probe] = &[ inherits any climb added there. The reading is the \ chain's operand count.", }, + Probe { + name: "cognitive/bash-test-connective-run", + lang: LANG::Bash, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Cognitive], + reading: |m| m.cognitive.cognitive_sum(), + }, + render: bash_test_connective_run, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1536: each `binary_expression` of a mis-nested \ + `[ … -o … ]` run asks where its boolean sequence ends, \ + which is the end of the outermost comparison of the \ + run. Climbing the run from every node for it, as \ + before the fix, takes this probe from 0.98 to 2.06 \ + (6.5 ms to 1.99 s at depth 4 000); the run is now \ + tracked top-down. The reading is one sequence per \ + switch between `-o` and `-a`.", + }, + Probe { + name: "abc/bash-arith-assign-chain", + lang: LANG::Bash, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc, Metric::Halstead], + reading: |m| m.halstead.total_operators(), + }, + render: bash_arith_assign_chain, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1536: Bash ABC once climbed from every `=` to see \ + whether a test holds it, which on a valid chained \ + arithmetic assignment is the whole spine per `=`. \ + Restoring that climb takes this probe from 1.00 to \ + 1.97 (0.79 ms to 72.5 ms at depth 4 000) and \ + `abc/bash-eq-and-test` to 1.91. ABC counts no \ + assignment for an arithmetic `=`, so the reading is \ + the Halstead operator count, which grows with the \ + depth.", + }, + Probe { + name: "abc/bash-eq-and-test", + lang: LANG::Bash, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc], + reading: |m| m.abc.conditions_sum(), + }, + render: bash_eq_and_test, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1536: the `[[ … = … && … ]]` misparse, where every \ + `=` is a comparison ABC counts. Each test now counts \ + its own from the top; the per-`=` climb it replaced \ + takes this probe from 1.02 to 1.91 (2.9 ms to 75.2 ms \ + at depth 4 000). The reading is one condition per \ + `=` plus the `if`.", + }, ]; #[cfg(test)] @@ -1556,6 +1821,10 @@ mod tests { use super::{Axis, PROBES, Probe, Workload}; + /// Probes whose shape deliberately fails to parse; see + /// [`shapes_parse_without_error`]. + const RECOVERY_PROBES: &[&str] = &["abc/perl-error-tail"]; + /// Deepest `tree_sitter` node depth reachable from the root. /// /// Iterative on an explicit stack: the shapes here nest thousands @@ -1641,16 +1910,34 @@ mod tests { /// parser limit — a recursion cap, a token-count ceiling — is /// reached at scale and not on a toy input, so a small fixed size /// answers a different question than the gate asks. + /// + /// The probes in [`RECOVERY_PROBES`] are the inverse: their shape + /// *is* error recovery, and the cost they price is reading the + /// `ERROR` root's children. A grammar that learned to parse one + /// would leave it timing an ordinary narrow tree, so each must still + /// recover into an `ERROR` root. #[test] fn shapes_parse_without_error() { for probe in PROBES { let size = probe.sizes[0]; let ast = parse(probe, size); + let root = ast.as_tree_sitter().root_node(); + let (parses_as_declared, declared) = if RECOVERY_PROBES.contains(&probe.name) { + (root.is_error(), "into an ERROR root") + } else { + (!root.has_error(), "without an ERROR node") + }; + let name = probe.name; + let head: String = (probe.render)(size).chars().take(200).collect(); assert!( - !ast.as_tree_sitter().root_node().has_error(), - "{}: shape must parse without an ERROR node at size {size}:\n{}…", - probe.name, - (probe.render)(size).chars().take(200).collect::(), + parses_as_declared, + "{name}: shape must parse {declared} at size {size}:\n{head}…" + ); + } + for name in RECOVERY_PROBES { + assert!( + PROBES.iter().any(|probe| probe.name == *name), + "RECOVERY_PROBES names {name}, which is not in PROBES", ); } } diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index ad11b8e7..923bd8a7 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -162,6 +162,13 @@ Read it as follows. | `cyclomatic/cpp-and-chain` | C++ | depth | the `&&` / `||` applied-operator gate (#1533) | linear | | `cognitive/cpp-and-chain` | C++ | depth | the boolean-sequence arm on the same chain (#1533) | linear | | `abc/cpp-and-chain` | C++ | depth | ABC's chain arm on the same chain (#1533) | linear | +| `abc/perl-or-chain` | Perl | depth | the misparse check's spine climb (#1273) | linear | +| `abc/perl-misparsed-comparison` | Perl | depth | the same climb on a chain the misparse absorbed (#1273) | linear | +| `abc/perl-and-statements` | Perl | width | the misparse check's operator lookup in a block (#1273) | linear | +| `abc/perl-error-tail` | Perl | width | `perl_binding_rewrites` under a flat `ERROR` root (#1540) | linear | +| `cognitive/bash-test-connective-run` | Bash | depth | the comparison-run end of a `[ … -o … ]` test (#1536) | linear | +| `abc/bash-arith-assign-chain` | Bash | depth | ABC's `=` comparison test on `(( a = … ))` (#1536) | linear | +| `abc/bash-eq-and-test` | Bash | depth | the same test on a `[[ … = … && … ]]` run (#1536) | linear | Four of these were quadratic when the harness landed, and they shared one cause: `tree_sitter` stores no parent pointer, so `Node::parent` @@ -365,6 +372,37 @@ depth 4 000 — while `cyclomatic/cpp-plus-chain`, the same chain through metric has its own probe because each reaches the chain through its own arm, and a climb added to one of them would leave the other two flat. +The seven Perl and Bash probes guard walks that went quadratic during +the #1273, #1536 and #1540 work and were caught only by a hand-timed +check in review ([#1575][perl-bash-probes]). Each was watched failing +with its pre-fix code put back behind a local patch, one walk at a +time: + +| Walk put back | Probe that fails | Clean | Regressed | Top rung | +|---|---|---|---|---| +| every Perl spine node climbs | `abc/perl-or-chain` | 1.00 | 2.00 | 8.8 ms → 2.33 s | +| (same patch) | `abc/perl-misparsed-comparison` | 0.99 | 2.01 | 8.8 ms → 2.35 s | +| the stop parent scanned from its start | `abc/perl-and-statements` | 1.08 | 2.01 | 3.7 ms → 306 ms | +| any parent read as a binding | `abc/perl-error-tail` | 1.19 | 2.05 | 2.9 ms → 492 ms | +| the Bash comparison run climbed per node | `cognitive/bash-test-connective-run` | 0.98 | 2.06 | 6.5 ms → 1.99 s | +| every Bash `=` climbs to its test | `abc/bash-arith-assign-chain` | 1.00 | 1.97 | 0.8 ms → 72.5 ms | +| (same patch) | `abc/bash-eq-and-test` | 1.02 | 1.91 | 2.9 ms → 75.2 ms | + +In each run only the named probes moved, and every other probe stayed +inside its bound. + +Removing only the kind gate in front of the Perl lookup moved nothing: +`abc/perl-and-statements` held at 1.19. On a block that gate is not +what keeps the walk linear, because the lookup seeks the child by byte. +That seek is `O(log n)` only where the grammar stores the children as +a balanced repeat, though, and an `ERROR` that recovery built by +skipping tokens is one flat array. So `abc/perl-error-tail` carries +only `=~` in its body. With a bareword or a `&&` chain added it fits +2.09 on today's tree ([#1584][flat-error]). It is also the one probe +whose shape is meant not to parse: `RECOVERY_PROBES` in `shapes.rs` +exempts it from the parse check and asserts that its root is still an +`ERROR`. + Treat the linear bounds above as covering the walk's ancestor *chain* threading, not every `O(depth)` lookup in the crate. @@ -461,6 +499,8 @@ a walk's chain bookkeeping, not just around a change to its cost. The [cfg-predicate]: https://github.com/dekobon/big-code-analysis/issues/1105 [attribute-row]: https://github.com/dekobon/big-code-analysis/issues/1431 [space-merge]: https://github.com/dekobon/big-code-analysis/issues/1106 +[perl-bash-probes]: https://github.com/dekobon/big-code-analysis/issues/1575 +[flat-error]: https://github.com/dekobon/big-code-analysis/issues/1584 The eleven control probes are what make the other readings mean something. @@ -524,7 +564,8 @@ alone. Add a `Probe` to `PROBES` in `big-code-analysis-bench/src/shapes.rs`. The unit tests in that module enforce what a probe has to satisfy: bytes affine in the size parameter, no parse errors at the smallest -declared size, a non-zero workload reading there, sizes that double, +declared size (or, for a probe listed in `RECOVERY_PROBES`, an `ERROR` +root), a non-zero workload reading there, sizes that double, and the invariant its `axis` selects — AST levels growing with the size on `Axis::Depth`, children growing while AST depth holds still on `Axis::Width`. Set `max_exponent` from a measurement on an idle host, From 9c372c578a679cb89cda17b4144cf10d24a8f526 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:48:17 -0700 Subject: [PATCH 16/34] perf(ast): read flat-ERROR siblings from the walk A seek by byte reaches a child in O(log n) only where the grammar stores the list as a repeat; an ERROR that recovery built by skipping tokens is one flat array, and no tree-sitter call reaches a child of one in better than its index. Perl ABC and Halstead ask for a sibling in such an ERROR once per child (the bareword and dash key tests, the misparse operator lookup, the key-sign test), so a file whose tail fails to parse was quadratic in the tail: 2.15 s at 16 000 copies of `{ last,`. Kotlin's `Ancestors::previous_sibling` scanned a string's children from its start, making `"$a $a ..."` quadratic on valid input (11.5 s at 8 000 names). The metric and ops walks now record the children of every node wider than 64 as they push them, in a SiblingCache they lend through Ancestors. Ancestors::siblings_after / siblings_before and previous_sibling binary-search that array; a narrow or unrecorded parent falls back to the seek, so every answer is unchanged. A debug assertion fails any walk that lends a cache without recording a wide node. Ancestors moves to node/ancestors.rs, keeping node.rs under its ploc limit. New probes halstead/perl-error-tail-keys and halstead/kotlin-short-interpolations, and abc/perl-error-tail widened to carry a bareword and a chain, all trip bench-scaling with the cache ignored (2.10, 1.74, 2.02) and hold at ~1.0 with it. Fixes #1584 --- .bca-baseline.toml | 10 +- big-code-analysis-ast/src/getter/kotlin.rs | 10 +- .../src/lang_helpers/perl.rs | 69 ++- big-code-analysis-ast/src/lib.rs | 2 +- big-code-analysis-ast/src/node.rs | 229 +------- big-code-analysis-ast/src/node/ancestors.rs | 549 ++++++++++++++++++ big-code-analysis-bench/src/shapes.rs | 112 +++- docs/development/benchmarking.md | 33 +- src/metrics/abc/perl.rs | 33 +- src/metrics/perl_keyword_key_tests.rs | 59 ++ src/ops.rs | 11 +- src/spaces.rs | 2 +- src/spaces/compute.rs | 6 +- 13 files changed, 816 insertions(+), 309 deletions(-) create mode 100644 big-code-analysis-ast/src/node/ancestors.rs diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 165af7df..4cf68e45 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -245,12 +245,6 @@ qualified = "IrulesCode::get_op_type" metric = "halstead.effort" value = 51477.285144213616 -[[entry]] -path = "big-code-analysis-ast/src/getter/java.rs" -qualified = "JavaCode::get_op_type" -metric = "halstead.effort" -value = 50584.467250345944 - [[entry]] path = "big-code-analysis-ast/src/getter/perl.rs" qualified = "PerlCode::get_op_type" @@ -1071,7 +1065,7 @@ value = 6.0 path = "src/ops.rs" qualified = "ops_inner" metric = "halstead.effort" -value = 70903.25755192844 +value = 84452.11438842113 [[entry]] path = "src/output/checkstyle.rs" @@ -1197,7 +1191,7 @@ value = 7.0 path = "src/spaces/compute.rs" qualified = "metrics_inner" metric = "halstead.effort" -value = 121169.85621346734 +value = 141885.98828207294 [[entry]] path = "src/spaces/compute.rs" diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index d177523a..b04fd312 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -157,14 +157,14 @@ fn kotlin_modifier_was_written_as_one<'a>(node: &Node<'a>, ancestors: Ancestors< // `function_value_parameters`, so the sibling is found without a scan // from the list's start: that made a long parameter list quadratic. fn kotlin_next_token_is_inserted<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - let mut current = *node; + let (mut current, mut current_ancestors) = (*node, ancestors); ancestors .iter(node) - .find_map(|(parent, _)| { - let next = parent - .children_after(¤t) + .find_map(|(parent, above)| { + let next = current_ancestors + .siblings_after(¤t) .find(|sibling| !sibling.as_tree_sitter().is_extra()); - current = parent; + (current, current_ancestors) = (parent, above); next }) .is_some_and(|mut next| { diff --git a/big-code-analysis-ast/src/lang_helpers/perl.rs b/big-code-analysis-ast/src/lang_helpers/perl.rs index c7494632..9b90f7e4 100644 --- a/big-code-analysis-ast/src/lang_helpers/perl.rs +++ b/big-code-analysis-ast/src/lang_helpers/perl.rs @@ -196,9 +196,9 @@ pub fn perl_bareword_is_key<'a>(word: &Node<'a>, ancestors: Ancestors<'a, '_>) - #[must_use] pub(crate) fn perl_dash_signs_key<'a>(dash: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { ancestors.iter(dash).next().is_some_and(|(parent, above)| { - perl_dash_signs_keyword_key(dash, &parent) + perl_dash_signs_keyword_key(dash, ancestors) || (parent.kind_id() == Perl::UnaryExpression as u16 - && next_non_extra(&parent, dash).is_some_and(|word| is_plain_bareword(&word)) + && next_non_extra(dash, ancestors).is_some_and(|word| is_plain_bareword(&word)) && is_key_position(&parent, above)) }) } @@ -225,25 +225,28 @@ pub(crate) fn perl_bareword_key_sign<'a>( perl_dash_signs_key(&dash, word_ancestors).then_some(dash) } -/// Whether `dash`, a Perl `-` token whose parent is `parent`, is the -/// sign of an auto-quoted `-and` or `-not` key (`$h{-not}`, -/// `(-and => 1)`), which the grammar splits into the `-` and the keyword -/// token. [`perl_and_is_operator`] and [`perl_not_is_key`] decide -/// whether the keyword is a key, and a key's sign is part of the string -/// whether or not a space separates the two. +/// Whether `dash`, a Perl `-` token whose ancestor chain is +/// `ancestors`, is the sign of an auto-quoted `-and` or `-not` key +/// (`$h{-not}`, `(-and => 1)`), which the grammar splits into the `-` +/// and the keyword token. [`perl_and_is_operator`] and +/// [`perl_not_is_key`] decide whether the keyword is a key, and a key's +/// sign is part of the string whether or not a space separates the two. #[must_use] -pub fn perl_dash_signs_keyword_key<'a>(dash: &Node<'a>, parent: &Node<'a>) -> bool { +pub fn perl_dash_signs_keyword_key<'a>(dash: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { // A sign is a prefix `-`, whose parent is the subscript's `ERROR` or // a `unary_expression`. An infix `-` is a real subtraction even // before an auto-quoted key (`(1 - and => 2)` subtracts the string), // and is by far the commonest `-`, so it is answered first. - if parent.kind_id() == Perl::BinaryExpression as u16 { + let Some(parent) = ancestors + .parent(dash) + .filter(|parent| parent.kind_id() != Perl::BinaryExpression as u16) + else { return false; - } + }; // `$h{-not}` keeps the keyword beside the `-`; `(-not => 1)` nests // it as the first child of the `-`'s operand. - next_non_extra(parent, dash).is_some_and(|next| { - keyword_is_key(&next, parent) + next_non_extra(dash, ancestors).is_some_and(|next| { + keyword_is_key(&next, &parent) || first_non_extra(&next).is_some_and(|keyword| keyword_is_key(&keyword, &next)) }) } @@ -257,17 +260,20 @@ pub fn perl_dash_signs_keyword_key<'a>(dash: &Node<'a>, parent: &Node<'a>) -> bo /// sign and its key. #[must_use] pub fn perl_key_is_signed<'a>(key: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - let mut up = ancestors.iter(key).map(|(ancestor, _)| ancestor); - up.next().is_some_and(|parent| { - let beside = parent - .children_before(key) + let mut up = ancestors.iter(key); + up.next().is_some_and(|(_, parent_ancestors)| { + let beside = ancestors + .siblings_before(key) .find(|sibling| !sibling.as_tree_sitter().is_extra()); + // A `-` beside the key shares its chain; the `operator` of the + // expression the key leads is a sibling of that expression. if let Some(dash) = beside.filter(|node| node.kind_id() == Perl::DASH as u16) { - return perl_dash_signs_keyword_key(&dash, &parent); + return perl_dash_signs_keyword_key(&dash, ancestors); } - up.next().is_some_and(|outer| { + up.next().is_some_and(|(outer, _)| { outer.child_by_field_name("operator").is_some_and(|dash| { - dash.kind_id() == Perl::DASH as u16 && perl_dash_signs_keyword_key(&dash, &outer) + dash.kind_id() == Perl::DASH as u16 + && perl_dash_signs_keyword_key(&dash, parent_ancestors) }) }) }) @@ -307,9 +313,9 @@ fn is_glued_word(word: &Node, test: &Node) -> bool { // token after the word. A key that stopped being the whole of its node // on the way up is no subscript key (`$h{1 - name}` is not quoted). fn is_key_position<'a>(key: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - let mut node = *key; + let (mut node, mut node_ancestors) = (*key, ancestors); let mut whole = true; - for (parent, _) in ancestors.iter(key) { + for (parent, above) in ancestors.iter(key) { if whole && matches!( parent.kind_id().into(), @@ -322,14 +328,14 @@ fn is_key_position<'a>(key: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { // grammar wraps a stray `,` (`(name, => 1)`) in an `ERROR` it // marks as an extra, and that `,`, not the word, is then the // token the `=>` quotes, which is nothing. - if let Some(next) = parent - .children_after(&node) + if let Some(next) = node_ancestors + .siblings_after(&node) .find(|sibling| sibling.is_error() || !sibling.as_tree_sitter().is_extra()) { return next.kind_id() == Perl::FatComma as u16; } whole &= parent.kind_id() == Perl::BinaryExpression as u16 && non_extra_count(&parent) == 1; - node = parent; + (node, node_ancestors) = (parent, above); } false } @@ -361,12 +367,13 @@ fn non_extra_count(node: &Node) -> usize { .count() } -// The sibling after `child` in `parent`, comments aside. Every key of -// a wide `(-a => 1, -b => 2, …)` list asks, so it is found without a -// scan from the list's start (#1545). -fn next_non_extra<'a>(parent: &Node<'a>, child: &Node<'a>) -> Option> { - parent - .children_after(child) +// The sibling after `node`, whose ancestor chain is `ancestors`, +// comments aside. Every key of a wide `(-a => 1, -b => 2, …)` list or +// a flat `ERROR` asks, so it is found without a scan from the parent's +// start (#1545, #1584). +fn next_non_extra<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> Option> { + ancestors + .siblings_after(node) .find(|sibling| !sibling.as_tree_sitter().is_extra()) } diff --git a/big-code-analysis-ast/src/lib.rs b/big-code-analysis-ast/src/lib.rs index 966a6e02..39c395af 100644 --- a/big-code-analysis-ast/src/lib.rs +++ b/big-code-analysis-ast/src/lib.rs @@ -137,7 +137,7 @@ pub use crate::getter::Getter; pub use crate::langs::*; pub use crate::languages::*; pub use crate::macros::ParseLangError; -pub use crate::node::{Ancestors, Node}; +pub use crate::node::{Ancestors, Node, SiblingCache}; pub use crate::parser::Parser; pub use crate::preproc::{ PreprocDiagnostic, PreprocFile, PreprocResults, fix_includes, get_macros, preprocess, diff --git a/big-code-analysis-ast/src/node.rs b/big-code-analysis-ast/src/node.rs index f7ce060b..3cd4c3b5 100644 --- a/big-code-analysis-ast/src/node.rs +++ b/big-code-analysis-ast/src/node.rs @@ -12,8 +12,11 @@ //! Tree-sitter wrappers: `Tree`, [`Node`], [`Ancestors`], and the cursors. +mod ancestors; mod parser_cache; +pub use ancestors::{AncestorIter, Ancestors, SiblingCache}; + use tree_sitter::Node as OtherNode; use tree_sitter::Tree as OtherTree; use tree_sitter::TreeCursor; @@ -511,8 +514,12 @@ impl<'a> Node<'a> { /// and nothing when `child` is not one of this node's children. /// /// That scan costs `child`'s index, so a walk that asks it of every - /// child of a wide list is quadratic in the list (#1545). This finds - /// `child` in `O(log n)` instead; see [`Node::children_before`]. + /// child of a wide list is quadratic in the list (#1545). This seeks + /// `child` by byte instead, which is `O(log n)` in a list the grammar + /// stores as a `repeat()` and still `O(index)` in a flat one, an + /// `ERROR` above all; see [`Cursor::goto_first_child_for_byte`]. A + /// walk that asks of every child of a node that may be flat goes + /// through [`Ancestors::siblings_after`] (#1584). pub fn children_after(&self, child: &Node<'a>) -> impl Iterator> + use<'a> { let mut cursor = self.cursor(); let mut more = @@ -529,8 +536,11 @@ impl<'a> Node<'a> { /// /// `child`'s index comes from [`Cursor::goto_first_child_for_byte`] /// and each sibling from [`Node::child`], both `O(log n)` in the - /// child count, for the reason the first gives: a `repeat()` list is - /// stored as a balanced tree of hidden nodes. + /// child count where the list is a `repeat()`, stored as a balanced + /// tree of hidden nodes, and both `O(index)` where it is flat, as an + /// `ERROR` built by skipping tokens is. A walk that asks of every + /// child of such a node goes through [`Ancestors::siblings_before`] + /// (#1584). pub fn children_before(&self, child: &Node<'a>) -> impl Iterator> + use<'a> { let node = *self; let index = self.seat_on_child(child, &mut self.cursor()).unwrap_or(0); @@ -663,212 +673,6 @@ impl<'a> Node<'a> { } } -/// The chain of a node's ancestors, root first, as recorded by a walker -/// that descended to that node. -/// -/// A predicate that asks a node for its parent pays [`Node::parent`]'s -/// `O(depth)` once per node, so `O(depth²)` over a deeply nested file, -/// however few parent steps it takes (#1084). A walker that visits -/// parents before children already holds the chain, and handing it down -/// turns each step into a slice index — the upward counterpart of the -/// downward flag propagation #1052 and #1062 used. -/// -/// Callers that reached a node some other way pass -/// [`Ancestors::unknown`], which climbs with [`Node::parent`]: the same -/// answers at the original cost. -#[derive(Clone, Copy, Debug)] -pub struct Ancestors<'tree, 'chain>(Option<&'chain [Node<'tree>]>); - -impl<'tree, 'chain> Ancestors<'tree, 'chain> { - /// No chain is available; every query climbs with [`Node::parent`]. - #[inline] - #[must_use] - pub const fn unknown() -> Self { - Self(None) - } - - /// `chain` lists every ancestor of the node about to be queried, - /// root first — so `chain.last()` is its parent and an empty chain - /// means the node is the root. - #[inline] - #[must_use] - pub const fn known(chain: &'chain [Node<'tree>]) -> Self { - Self(Some(chain)) - } - - /// [`Ancestors::known`], but first checks that `chain` really is - /// `node`'s ancestry. - /// - /// The parity test below proves the walker's truncate/push rule on a - /// *replica* walker, so it cannot see `metrics_inner` itself - /// desynchronising — a `chain.push` moved ahead of the per-node - /// computes, say, or a `continue` inserted above the truncate. - /// [`Ancestors::parent`] trusts `chain.last()` unvalidated, so such a - /// drift would feed every predicate a wrong ancestor silently rather - /// than fail. Walkers that maintain a chain should construct through - /// here; [`Ancestors::known`] stays unchecked for the callers that - /// deliberately pair a chain with a foreign node. - /// - /// # Two checks, because the exact one is not affordable by default - /// - /// The invariant is "`chain.last()` **is** `node.parent()`", and - /// asking that outright costs [`Node::parent`]'s `O(depth)` — the - /// very lookup #1084 exists to remove. Per node, on every walk that - /// constructs a checked chain, it made every debug-build walk - /// `O(nodes × depth)` while the shipped walk is `O(nodes)`: a tax on - /// every `cargo test`, worst on the deep-nesting regression tests - /// that exist to pin the shipped walk's linearity (#1122). It now - /// runs only under `--cfg chain_audit`, which `make chain-audit` and - /// the CI lane of the same name set — and there as a plain - /// `assert_eq!`, so the audit has teeth in a release profile too. - /// - /// What stays on in every debug build is an `O(1)` *consequence* of - /// that invariant: a parent's byte span contains its child's, and no - /// node is its own parent. Strictly weaker — a grandparent contains - /// the node as well — but it is shaped to the two ways a walker - /// desynchronises. A `push` moved ahead of the per-node computes - /// leaves `chain.last() == node`; a dropped `truncate` leaves the - /// previous subtree's path, whose last entry is disjoint from the - /// node that follows it. Both trip here, on the first node that - /// shows them, at four integer comparisons. - #[must_use] - pub fn checked(chain: &'chain [Node<'tree>], node: &Node<'tree>) -> Self { - // Opt-in only: `Node::parent` restarts at the root, so this is - // the `O(nodes × depth)` walk described above. - #[cfg(chain_audit)] - assert_eq!( - chain.last().map(Node::id), - node.parent().map(|parent| parent.id()), - "ancestor chain desynchronised on a {} node", - node.kind() - ); - debug_assert!( - chain.last().is_none_or(|parent| { - parent.id() != node.id() - && parent.start_byte() <= node.start_byte() - && node.end_byte() <= parent.end_byte() - }), - "ancestor chain desynchronised on a {} node: chain.last() neither \ - contains it nor differs from it", - node.kind() - ); - Self::known(chain) - } - - /// How far the node this chain describes sits from the root, or - /// `None` when no chain is known — deriving it then would cost the - /// [`Node::parent`] climb the chain exists to remove. - #[inline] - pub fn depth(self) -> Option { - self.0.map(<[Node<'tree>]>::len) - } - - /// `node`'s parent. - #[inline] - #[must_use] - pub fn parent(self, node: &Node<'tree>) -> Option> { - match self.0 { - Some(chain) => chain.last().copied(), - None => node.parent(), - } - } - - /// Whether `node`'s parent has kind id `kind`. - /// - /// The compound-leaf guard of `.claude/rules/grammar-dispatch.md` - /// section 5 asks this and nothing else: a delimiter or keyword - /// token is suppressed *only* directly under the construct that - /// owns it, never under an arbitrary ancestor. Eleven dispatch - /// arms across `getter`, `checker` and the metric walkers spelled - /// it out as `.parent(node).is_some_and(|p| p.kind_id() == X as - /// u16)`, which wraps onto four rustfmt lines inside a match guard - /// and buries the question under the plumbing; #1314 added six more - /// and folded all seventeen onto this. Three sites spelling their - /// binding `|parent|` were left alone and would each fit: - /// `impl_is_else_if_parent_clause!` (`src/checker.rs`), - /// `is_stabby_lambda_body` (`src/checker/ruby.rs`) and - /// `is_useful_comment` (`src/checker/rust.rs`, a let-chain). - /// - /// A `false` when `node` has no parent is the answer every one of - /// those call sites wants: a root node's token is not inside the - /// construct, so it is not suppressed. - #[inline] - #[must_use] - pub fn parent_has_kind(self, node: &Node<'tree>, kind: u16) -> bool { - self.parent(node).is_some_and(|p| p.kind_id() == kind) - } - - /// The sibling immediately before `node`, or `None` when `node` is - /// its parent's first child or has no parent. - /// - /// `tree_sitter`'s own `prev_sibling` resolves the parent first - /// (`ts_node__prev_sibling` opens with `ts_node_parent`), so it - /// carries the same `O(depth)` cost [`Node::parent`] does. With a - /// known chain the parent is free and what remains is a cursor walk - /// over the siblings. - #[inline] - #[must_use] - pub fn previous_sibling(self, node: &Node<'tree>) -> Option> { - let Some(chain) = self.0 else { - return node.previous_sibling(); - }; - let parent = chain.last()?; - let mut previous = None; - for child in parent.children() { - if child.id() == node.id() { - return previous; - } - previous = Some(child); - } - // `node` is not among `parent.children()`, so this chain does - // not describe `node`. Answering `None` would claim "no - // previous sibling", which is a different (and wrong) answer; - // fall back to the authoritative lookup instead. - node.previous_sibling() - } - - /// `node`'s ancestors, nearest first, each paired with *its* own - /// ancestry so a predicate applied to an ancestor stays as cheap as - /// one applied to `node`. - #[inline] - #[must_use] - pub fn iter(self, node: &Node<'tree>) -> AncestorIter<'tree, 'chain> { - match self.0 { - Some(chain) => AncestorIter::Chain(chain), - None => AncestorIter::Climb(node.parent()), - } - } -} - -/// Ancestor iterator returned by [`Ancestors::iter`], nearest first. -pub enum AncestorIter<'tree, 'chain> { - /// The not-yet-yielded prefix of a known chain. Its last element is - /// the next ancestor, and the prefix before it is that ancestor's - /// own chain — so splitting from the back hands out both at once. - Chain(&'chain [Node<'tree>]), - /// The next ancestor to yield, reached by climbing. - Climb(Option>), -} - -impl<'tree, 'chain> Iterator for AncestorIter<'tree, 'chain> { - type Item = (Node<'tree>, Ancestors<'tree, 'chain>); - - fn next(&mut self) -> Option { - match self { - Self::Chain(remaining) => { - let (&nearest, above) = remaining.split_last()?; - *remaining = above; - Some((nearest, Ancestors::known(above))) - } - Self::Climb(next) => { - let nearest = (*next)?; - *next = nearest.parent(); - Some((nearest, Ancestors::unknown())) - } - } - } -} - /// Pre-order iterator over a node and its descendants, returned by /// [`Node::preorder`]. /// @@ -942,6 +746,11 @@ impl<'a> Cursor<'a> { /// is `O(log n)` in the sibling count where `children().position(..)` /// is `O(n)` — the difference between a linear and a quadratic walk /// when every child of a wide list asks for its own index. + /// + /// Only for such a list. The seek steps through the children in + /// order and can skip only a *hidden* subtree, so over a flat child + /// array — an `ERROR` that recovery built by skipping tokens has no + /// hidden levels — it is the same `O(index)` scan (#1584). #[inline] pub fn goto_first_child_for_byte(&mut self, byte: usize) -> Option { self.0.goto_first_child_for_byte(byte) diff --git a/big-code-analysis-ast/src/node/ancestors.rs b/big-code-analysis-ast/src/node/ancestors.rs new file mode 100644 index 00000000..01b28608 --- /dev/null +++ b/big-code-analysis-ast/src/node/ancestors.rs @@ -0,0 +1,549 @@ +//! [`Ancestors`], the chain of a node's ancestors a walker threads down +//! to its predicates, and the [`SiblingCache`] it can lend with it. + +use super::Node; + +/// The chain of a node's ancestors, root first, as recorded by a walker +/// that descended to that node. +/// +/// A predicate that asks a node for its parent pays [`Node::parent`]'s +/// `O(depth)` once per node, so `O(depth²)` over a deeply nested file, +/// however few parent steps it takes (#1084). A walker that visits +/// parents before children already holds the chain, and handing it down +/// turns each step into a slice index — the upward counterpart of the +/// downward flag propagation #1052 and #1062 used. +/// +/// Callers that reached a node some other way pass +/// [`Ancestors::unknown`], which climbs with [`Node::parent`]: the same +/// answers at the original cost. +#[derive(Clone, Copy, Debug)] +pub struct Ancestors<'tree, 'chain> { + chain: Option<&'chain [Node<'tree>]>, + siblings: Option<&'chain SiblingCache<'tree>>, +} + +impl<'tree, 'chain> Ancestors<'tree, 'chain> { + /// No chain is available; every query climbs with [`Node::parent`]. + #[inline] + #[must_use] + pub const fn unknown() -> Self { + Self { + chain: None, + siblings: None, + } + } + + /// `chain` lists every ancestor of the node about to be queried, + /// root first — so `chain.last()` is its parent and an empty chain + /// means the node is the root. + #[inline] + #[must_use] + pub const fn known(chain: &'chain [Node<'tree>]) -> Self { + Self { + chain: Some(chain), + siblings: None, + } + } + + /// This chain, answering [`siblings_after`](Self::siblings_after) and + /// [`siblings_before`](Self::siblings_before) through `cache`, which + /// the walker that keeps the chain owns for the whole walk. + #[inline] + #[must_use] + pub const fn with_siblings(self, cache: &'chain SiblingCache<'tree>) -> Self { + Self { + siblings: Some(cache), + ..self + } + } + + /// [`Ancestors::known`], but first checks that `chain` really is + /// `node`'s ancestry. + /// + /// The parity test below proves the walker's truncate/push rule on a + /// *replica* walker, so it cannot see `metrics_inner` itself + /// desynchronising — a `chain.push` moved ahead of the per-node + /// computes, say, or a `continue` inserted above the truncate. + /// [`Ancestors::parent`] trusts `chain.last()` unvalidated, so such a + /// drift would feed every predicate a wrong ancestor silently rather + /// than fail. Walkers that maintain a chain should construct through + /// here; [`Ancestors::known`] stays unchecked for the callers that + /// deliberately pair a chain with a foreign node. + /// + /// # Two checks, because the exact one is not affordable by default + /// + /// The invariant is "`chain.last()` **is** `node.parent()`", and + /// asking that outright costs [`Node::parent`]'s `O(depth)` — the + /// very lookup #1084 exists to remove. Per node, on every walk that + /// constructs a checked chain, it made every debug-build walk + /// `O(nodes × depth)` while the shipped walk is `O(nodes)`: a tax on + /// every `cargo test`, worst on the deep-nesting regression tests + /// that exist to pin the shipped walk's linearity (#1122). It now + /// runs only under `--cfg chain_audit`, which `make chain-audit` and + /// the CI lane of the same name set — and there as a plain + /// `assert_eq!`, so the audit has teeth in a release profile too. + /// + /// What stays on in every debug build is an `O(1)` *consequence* of + /// that invariant: a parent's byte span contains its child's, and no + /// node is its own parent. Strictly weaker — a grandparent contains + /// the node as well — but it is shaped to the two ways a walker + /// desynchronises. A `push` moved ahead of the per-node computes + /// leaves `chain.last() == node`; a dropped `truncate` leaves the + /// previous subtree's path, whose last entry is disjoint from the + /// node that follows it. Both trip here, on the first node that + /// shows them, at four integer comparisons. + #[must_use] + pub fn checked(chain: &'chain [Node<'tree>], node: &Node<'tree>) -> Self { + // Opt-in only: `Node::parent` restarts at the root, so this is + // the `O(nodes × depth)` walk described above. + #[cfg(chain_audit)] + assert_eq!( + chain.last().map(Node::id), + node.parent().map(|parent| parent.id()), + "ancestor chain desynchronised on a {} node", + node.kind() + ); + debug_assert!( + chain.last().is_none_or(|parent| { + parent.id() != node.id() + && parent.start_byte() <= node.start_byte() + && node.end_byte() <= parent.end_byte() + }), + "ancestor chain desynchronised on a {} node: chain.last() neither \ + contains it nor differs from it", + node.kind() + ); + Self::known(chain) + } + + /// How far the node this chain describes sits from the root, or + /// `None` when no chain is known — deriving it then would cost the + /// [`Node::parent`] climb the chain exists to remove. + #[inline] + pub fn depth(self) -> Option { + self.chain.map(<[Node<'tree>]>::len) + } + + /// `node`'s parent. + #[inline] + #[must_use] + pub fn parent(self, node: &Node<'tree>) -> Option> { + match self.chain { + Some(chain) => chain.last().copied(), + None => node.parent(), + } + } + + /// Whether `node`'s parent has kind id `kind`. + /// + /// The compound-leaf guard of `.claude/rules/grammar-dispatch.md` + /// section 5 asks this and nothing else: a delimiter or keyword + /// token is suppressed *only* directly under the construct that + /// owns it, never under an arbitrary ancestor. Eleven dispatch + /// arms across `getter`, `checker` and the metric walkers spelled + /// it out as `.parent(node).is_some_and(|p| p.kind_id() == X as + /// u16)`, which wraps onto four rustfmt lines inside a match guard + /// and buries the question under the plumbing; #1314 added six more + /// and folded all seventeen onto this. Three sites spelling their + /// binding `|parent|` were left alone and would each fit: + /// `impl_is_else_if_parent_clause!` (`src/checker.rs`), + /// `is_stabby_lambda_body` (`src/checker/ruby.rs`) and + /// `is_useful_comment` (`src/checker/rust.rs`, a let-chain). + /// + /// A `false` when `node` has no parent is the answer every one of + /// those call sites wants: a root node's token is not inside the + /// construct, so it is not suppressed. + #[inline] + #[must_use] + pub fn parent_has_kind(self, node: &Node<'tree>, kind: u16) -> bool { + self.parent(node).is_some_and(|p| p.kind_id() == kind) + } + + /// The sibling immediately before `node`, or `None` when `node` is + /// its parent's first child or has no parent. + /// + /// `tree_sitter`'s own `prev_sibling` resolves the parent first + /// (`ts_node__prev_sibling` opens with `ts_node_parent`), so it + /// carries the same `O(depth)` cost [`Node::parent`] does. With a + /// known chain the parent is free and what remains is a cursor walk + /// over the siblings — or, for a wide parent the walk recorded in its + /// [`SiblingCache`], a binary search, since a walk over every child + /// of a Kotlin string's `$name` run was quadratic in it (#1584). + #[inline] + #[must_use] + pub fn previous_sibling(self, node: &Node<'tree>) -> Option> { + let Some(chain) = self.chain else { + return node.previous_sibling(); + }; + let parent = chain.last()?; + if let Lookup::Wide(children, index) = self.lookup(node) { + return match index { + Some(index) => children[..index].last().copied(), + None => node.previous_sibling(), + }; + } + let mut previous = None; + for child in parent.children() { + if child.id() == node.id() { + return previous; + } + previous = Some(child); + } + // `node` is not among `parent.children()`, so this chain does + // not describe `node`. Answering `None` would claim "no + // previous sibling", which is a different (and wrong) answer; + // fall back to the authoritative lookup instead. + node.previous_sibling() + } + + /// `node`'s ancestors, nearest first, each paired with *its* own + /// ancestry so a predicate applied to an ancestor stays as cheap as + /// one applied to `node`. + #[inline] + #[must_use] + pub fn iter(self, node: &Node<'tree>) -> AncestorIter<'tree, 'chain> { + match self.chain { + Some(chain) => AncestorIter::Chain(chain, self.siblings), + None => AncestorIter::Climb(node.parent()), + } + } + + /// The siblings after `node`, nearest first: what + /// [`Node::children_after`] yields on `node`'s parent, and nothing + /// for the root. `self` is `node`'s chain. + /// + /// The seek [`Node::children_after`] runs is `O(index)` over a flat + /// child array, so a walk that asks of every child of a wide + /// `ERROR` is quadratic in it (#1584). A walk that lends a + /// [`SiblingCache`] has already collected a wide parent's children, + /// and the lookup is a binary search among them. + pub fn siblings_after( + self, + node: &Node<'tree>, + ) -> impl Iterator> + use<'tree, 'chain> { + let (wide, narrow) = match self.lookup(node) { + Lookup::Wide(children, index) => (index.map(|index| &children[index + 1..]), None), + Lookup::Narrow(parent) => (None, parent.map(|parent| parent.children_after(node))), + }; + let wide = wide.into_iter().flatten().copied(); + wide.chain(narrow.into_iter().flatten()) + } + + /// The siblings before `node`, nearest first: what + /// [`Node::children_before`] yields on `node`'s parent, and nothing + /// for the root. `self` is `node`'s chain; see + /// [`siblings_after`](Self::siblings_after). + pub fn siblings_before( + self, + node: &Node<'tree>, + ) -> impl Iterator> + use<'tree, 'chain> { + let (wide, narrow) = match self.lookup(node) { + Lookup::Wide(children, index) => (index.map(|index| &children[..index]), None), + Lookup::Narrow(parent) => (None, parent.map(|parent| parent.children_before(node))), + }; + let wide = wide.into_iter().flatten().rev().copied(); + wide.chain(narrow.into_iter().flatten()) + } + + // Where `node`'s siblings are read from: its parent's children as the + // walk recorded them and `node`'s index among them, or the parent to + // seek in when the walk recorded none. + fn lookup(self, node: &Node<'tree>) -> Lookup<'tree, 'chain> { + let parent = self.parent(node); + let recorded = match (self.siblings, self.chain, parent) { + (Some(cache), Some(chain), Some(parent)) => { + let recorded = cache.children(&parent, chain.len() - 1); + // A walk that stopped recording would answer the same, + // only quadratically, which no metric value shows. + debug_assert!( + recorded.is_some() || parent.child_count() <= WIDE_PARENT, + "a walk lending a SiblingCache must record every wide node on its \ + chain (#1584), but not this {}", + parent.kind() + ); + recorded + } + _ => None, + }; + match recorded { + Some(children) => Lookup::Wide(children, position_by_start(children, node)), + None => Lookup::Narrow(parent), + } + } +} + +enum Lookup<'tree, 'chain> { + Wide(&'chain [Node<'tree>], Option), + Narrow(Option>), +} + +// Children past which a walk records a node's children for sibling +// lookups. A seek over this many is a short scan however the grammar +// stores them, so recording a narrower node would copy it for nothing. +const WIDE_PARENT: usize = 64; + +// `node`'s index in `children`, a parent's children in order. Their start +// bytes never decrease, so a binary search lands on the first child +// starting where `node` does; only zero-width children, which recovery +// inserts, can share that byte with it. +fn position_by_start(children: &[Node], node: &Node) -> Option { + let first = children.partition_point(|child| child.start_byte() < node.start_byte()); + children[first..] + .iter() + .take_while(|child| child.start_byte() == node.start_byte()) + .position(|child| child.id() == node.id()) + .map(|offset| first + offset) +} + +/// The children of each wide node on a walk's chain, recorded as the +/// walk pushes them, so that [`Ancestors::siblings_after`] and +/// [`Ancestors::siblings_before`] find a sibling in a flat `ERROR` +/// without a scan from its start (#1584). +/// +/// One slot per depth, keyed by the node's id. A walk descends into a +/// node only after pushing its children, so every wide node on the +/// chain has its slot; a slot left behind by a node the walk has moved +/// past holds another id and is not read. Recording copies what the +/// walk already holds, once per wide node. +#[derive(Debug, Default)] +pub struct SiblingCache<'tree>(Vec<(usize, Vec>)>); + +impl<'tree> SiblingCache<'tree> { + /// Records `children`, in order, as the children of `node`, which + /// sits at `depth` in the walk's chain. A node of no more than a + /// short scan's children is not recorded. + pub fn record( + &mut self, + node: &Node<'tree>, + depth: usize, + children: impl IntoIterator>, + ) { + if node.child_count() <= WIDE_PARENT { + return; + } + if self.0.len() <= depth { + self.0.resize_with(depth + 1, Default::default); + } + // An unused slot's id is 0, which no node has: tree-sitter's id + // is the address of the node's subtree. + let (id, recorded) = &mut self.0[depth]; + *id = node.id(); + recorded.clear(); + recorded.extend(children); + } + + // `parent`'s children, when the walk recorded them at `depth`. + fn children(&self, parent: &Node<'tree>, depth: usize) -> Option<&[Node<'tree>]> { + self.0 + .get(depth) + .filter(|(id, _)| *id == parent.id()) + .map(|(_, children)| children.as_slice()) + } +} + +/// Ancestor iterator returned by [`Ancestors::iter`], nearest first. +pub enum AncestorIter<'tree, 'chain> { + /// The not-yet-yielded prefix of a known chain, and the walk's + /// [`SiblingCache`] if it lent one. The prefix's last element is + /// the next ancestor, and the prefix before it is that ancestor's + /// own chain — so splitting from the back hands out both at once. + Chain(&'chain [Node<'tree>], Option<&'chain SiblingCache<'tree>>), + /// The next ancestor to yield, reached by climbing. + Climb(Option>), +} + +impl<'tree, 'chain> Iterator for AncestorIter<'tree, 'chain> { + type Item = (Node<'tree>, Ancestors<'tree, 'chain>); + + fn next(&mut self) -> Option { + match self { + Self::Chain(remaining, siblings) => { + let (&nearest, above) = remaining.split_last()?; + *remaining = above; + let ancestors = Ancestors { + chain: Some(above), + siblings: *siblings, + }; + Some((nearest, ancestors)) + } + Self::Climb(next) => { + let nearest = (*next)?; + *next = nearest.parent(); + Some((nearest, Ancestors::unknown())) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::node::Tree; + + // The ids of `nodes`, so a failing comparison prints which differ. + #[cfg(feature = "perl")] + fn ids<'a>(nodes: impl Iterator>) -> Vec { + nodes.map(|node| node.id()).collect() + } + + // `parent`'s children through `child(i)`, the raw walk that neither + // the seek nor the recorded array is built on. + #[cfg(feature = "perl")] + fn raw_children(parent: &Node) -> Vec { + let raw = parent.as_tree_sitter(); + (0..raw.child_count() as u32) + .filter_map(|i| raw.child(i)) + .map(|child| child.id()) + .collect() + } + + // `my @r = (` and a body never closed: recovery makes the root an + // `ERROR` whose children are the body's tokens, one flat array of + // well over `WIDE_PARENT` of them. + #[cfg(feature = "perl")] + fn flat_error() -> Tree { + let source = format!("my @r = ({}", "{\nlast,\n$a && $b,\n".repeat(20)); + Tree::new::(source.as_bytes()) + } + + /// A recorded wide parent answers every child's siblings, and its + /// previous sibling, from the recorded array, and answers exactly + /// what the raw walk does on either side of it — the `ERROR` whose + /// flat array made the seek `O(index)` (#1584). A node that is no + /// child of it gets nothing. + #[cfg(feature = "perl")] + #[test] + fn siblings_of_a_recorded_wide_parent_match_the_child_walk() { + let tree = flat_error(); + let root = tree.get_root(); + assert!(root.is_error(), "the fixture's tail must fail to parse"); + assert!(root.child_count() > WIDE_PARENT, "the ERROR must be wide"); + + let mut cache = SiblingCache::default(); + cache.record(&root, 0, root.children()); + let chain = [root]; + let ancestors = Ancestors::known(&chain).with_siblings(&cache); + let all = raw_children(&root); + + let (mut observed, mut expected) = (Vec::new(), Vec::new()); + for (index, child) in root.children().enumerate() { + let recorded_index = match ancestors.lookup(&child) { + Lookup::Wide(_, index) => index, + Lookup::Narrow(_) => None, + }; + observed.push(( + index, + recorded_index, + ids(ancestors.siblings_after(&child)), + ids(ancestors.siblings_before(&child)), + ancestors.previous_sibling(&child).map(|node| node.id()), + )); + expected.push(( + index, + Some(index), + all[index + 1..].to_vec(), + all[..index].iter().rev().copied().collect(), + index.checked_sub(1).map(|previous| all[previous]), + )); + } + assert_eq!(observed, expected); + + // A grandchild starting where its parent does, so the search + // reaches it and only the id comparison turns it away. + let stray = root + .children() + .find_map(|child| { + child + .child(0) + .filter(|g| g.start_byte() == child.start_byte()) + }) + .expect("the fixture nests a node at its parent's start"); + assert!(matches!(ancestors.lookup(&stray), Lookup::Wide(_, None))); + assert_eq!(ancestors.siblings_after(&stray).count(), 0); + assert_eq!(ancestors.siblings_before(&stray).count(), 0); + // `previous_sibling` falls back to the node's own lookup instead, + // which finds a stray's real previous sibling: `$b`'s is the `&&` + // of its own chain, not anything in the root. + let operand = root + .children() + .find_map(|child| child.child(2)) + .expect("the fixture holds `$a && $b`"); + let previous = ancestors + .previous_sibling(&operand) + .expect("the fallback finds the `&&`"); + assert_eq!(previous.kind(), "&&"); + } + + /// A narrow parent is never recorded, and its siblings come from + /// the seek — with or without a cache lent. + #[cfg(feature = "perl")] + #[test] + fn a_narrow_parent_is_sought_not_recorded() { + let tree = flat_error(); + let root = tree.get_root(); + let chain = [root]; + let narrow = root + .children() + .find(|child| (2..=WIDE_PARENT).contains(&child.child_count())) + .expect("the fixture holds a narrow node with several children"); + let mut cache = SiblingCache::default(); + cache.record(&narrow, 1, narrow.children()); + assert!(cache.0.is_empty(), "a narrow node must not be recorded"); + + let narrow_chain = [root, narrow]; + let all = raw_children(&narrow); + for ancestors in [ + Ancestors::known(&narrow_chain).with_siblings(&cache), + Ancestors::known(&narrow_chain), + ] { + let first = narrow.child(0).expect("narrow node has children"); + assert!(matches!(ancestors.lookup(&first), Lookup::Narrow(Some(_)))); + assert_eq!(ids(ancestors.siblings_after(&first)), all[1..]); + } + // Without a cache a wide parent is sought too. + let first = root.child(0).expect("root has children"); + assert!(matches!( + Ancestors::known(&chain).lookup(&first), + Lookup::Narrow(Some(_)) + )); + } + + /// A walk that lends a cache but did not record a wide node on its + /// chain would answer correctly and quadratically; the assertion is + /// what makes that visible to every test that walks one. + #[cfg(all(feature = "perl", debug_assertions))] + #[test] + #[should_panic(expected = "must record every wide node on its chain")] + fn an_unrecorded_wide_parent_trips_the_walk_assertion() { + let tree = flat_error(); + let root = tree.get_root(); + let cache = SiblingCache::default(); + let chain = [root]; + let first = root.child(0).expect("root has children"); + let _ = Ancestors::known(&chain) + .with_siblings(&cache) + .lookup(&first); + } + + /// The binary search finds every child at its index, including a + /// zero-width one: recovery inserts it at the byte its successor + /// starts at, so only the id tells the two apart. The Kotlin + /// fixture's `vararg: Int` holds one. + #[cfg(feature = "kotlin")] + #[test] + fn position_by_start_finds_a_zero_width_child() { + let tree = Tree::new::(b"fun f(vararg: Int, b: Int) {}"); + let mut zero_width = 0; + for node in tree.get_root().preorder() { + let children: Vec<_> = node.children().collect(); + for (index, child) in children.iter().enumerate() { + zero_width += usize::from(child.start_byte() == child.end_byte()); + assert_eq!(position_by_start(&children, child), Some(index)); + } + } + assert!( + zero_width > 0, + "fixture must hold an inserted zero-width child" + ); + } +} diff --git a/big-code-analysis-bench/src/shapes.rs b/big-code-analysis-bench/src/shapes.rs index 9abfa256..f75a23c6 100644 --- a/big-code-analysis-bench/src/shapes.rs +++ b/big-code-analysis-bench/src/shapes.rs @@ -537,6 +537,18 @@ pub fn perl_pattern_list(width: usize) -> String { format!("my @r = ({});\n", vec!["/a/"; width].join(", ")) } +/// Kotlin: `val s = "$a $a …"` — one string holding `width` short +/// `$name` interpolations. +/// +/// The width shape for Kotlin Halstead's short-interpolation test, which +/// reads the token before every name. It found that token by scanning +/// the string's children from its start, quadratic in the name count on +/// valid input: 11.5 s for 8 000 names in a release build (#1584). +#[must_use] +pub fn kotlin_short_interpolations(width: usize) -> String { + format!("val s = \"{}\"\n", "$a ".repeat(width)) +} + /// Kotlin: `inline fun f(noinline p0: () -> Unit, …) {}` — one function /// with `width` modified parameters. /// @@ -597,25 +609,35 @@ pub fn perl_and_statements(width: usize) -> String { format!("sub f {{ {}}}\n", "$a && $b; ".repeat(width)) } -/// Perl: `my @r = (` followed by `width` copies of `$d $c =~ /x/;⏎{⏎`, -/// never closed. +/// Perl: `my @r = (` followed by `width` copies of +/// `last if $c =~ /x/;⏎{⏎$a && $b,⏎`, never closed. /// /// The width shape for a file whose tail fails to parse, as /// `perl5db.pl`'s does: tree-sitter-perl gives up on the unclosed list, -/// the root becomes an `ERROR`, and every `=~` in the tail is its direct -/// child. Perl ABC asks each `=~` whether it binds a rewrite, which -/// reads its parent's last operand; read off the `ERROR`, that is a -/// scan of the whole tail per `=~`. The body is the smallest found that -/// keeps the `=~` flat: without the stray `$d` and `{` it folds back -/// into a narrow `binary_expression`. -/// -/// FIXME(#1584): a bareword or a `&&` chain in this body makes the probe -/// quadratic on today's tree, because a flat `ERROR` defeats the -/// `O(log n)` child seek `Node::children_after` / `children_before` -/// rely on. Add `last if` and `$a && $b,` to the body with that fix. +/// the root becomes an `ERROR`, and the tail's tokens are its direct +/// children, one flat array. Perl ABC asks of each `=~` whether it binds +/// a rewrite, of each bareword (`last`) whether a `=>` follows it, and +/// of each `&&` chain which operator precedes it. Each question reads a +/// neighbour in the `ERROR`, and a seek by byte over a flat array scans +/// it from its start, so each was `O(width)` until the walk recorded the +/// `ERROR`'s children (#1584). #[must_use] pub fn perl_error_tail(width: usize) -> String { - format!("my @r = ({}", "$d $c =~ /x/;\n{\n".repeat(width)) + format!( + "my @r = ({}", + "last if $c =~ /x/;\n{\n$a && $b,\n".repeat(width) + ) +} + +/// Perl: `my @r = (` followed by `width` copies of +/// `{⏎- and,⏎not,⏎-a => 1,⏎`, never closed. +/// +/// [`perl_error_tail`]'s flat `ERROR`, holding the key shapes Halstead +/// asks about: whether a `=>` follows each `-a`, and whether a `-` signs +/// each `and` / `not` key. +#[must_use] +pub fn perl_error_tail_keys(width: usize) -> String { + format!("my @r = ({}", "{\n- and,\nnot,\n-a => 1,\n".repeat(width)) } /// Bash: `if [ "$a" = 0 -o "$a" = 0 -a … ]; then :; fi`, the @@ -1613,16 +1635,56 @@ pub const PROBES: &[Probe] = &[ render: perl_error_tail, sizes: LINEAR_WIDTHS, max_exponent: LINEAR_BOUND, - rationale: "`perl_binding_rewrites` asks each `=~` whether it \ - binds an `s///` / `tr///` by reading its parent's last \ - operand. In a file whose tail fails to parse, the \ - parent is an `ERROR` holding the whole tail, as in \ - `perl5db.pl`. Dropping the check that only a \ - `binary_expression` binds takes this probe from 1.19 \ - to 2.05 (2.9 ms to 492 ms at width 2 000). The only \ - probe whose shape does not parse (`RECOVERY_PROBES`). \ - The reading is two conditions per `=~` line, the \ - match and its pattern.", + rationale: "In a file whose tail fails to parse, as in \ + `perl5db.pl`, every `=~`, bareword and `&&` chain of \ + the tail is a child of one flat `ERROR`. \ + `perl_binding_rewrites` reads each `=~`'s parent's \ + last operand, the bareword-key test the token after \ + each `last`, and the misparse check the token before \ + each chain. Reading those off the `ERROR` from its \ + start made this probe fit 2.09 (#1540, #1584). A \ + shape that does not parse (`RECOVERY_PROBES`). The \ + reading is four conditions per copy of the body.", + }, + Probe { + name: "halstead/perl-error-tail-keys", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Halstead], + reading: |m| m.halstead.total_operands(), + }, + render: perl_error_tail_keys, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "`abc/perl-error-tail`'s flat `ERROR`, holding the \ + `-and`, `not` and `-a` keys Halstead bills: the key \ + test asks for the token after each, and the sign \ + test for the token beside it. Read off the `ERROR` \ + from its start, as before #1584, both were \ + quadratic in the tail. The reading is three operands \ + per copy of the body.", + }, + Probe { + name: "halstead/kotlin-short-interpolations", + lang: LANG::Kotlin, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Halstead], + reading: |m| m.halstead.total_operands(), + }, + render: kotlin_short_interpolations, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "Kotlin Halstead asks of every `$name` whether a `$` \ + token precedes it. `Ancestors::previous_sibling` \ + answered by scanning the string's children from its \ + start, quadratic in the name count on valid input, \ + until it read the children the walk recorded \ + (#1584). The reading is one operand per name plus \ + `s`.", }, Probe { name: "halstead/kotlin-noinline-params", @@ -1823,7 +1885,7 @@ mod tests { /// Probes whose shape deliberately fails to parse; see /// [`shapes_parse_without_error`]. - const RECOVERY_PROBES: &[&str] = &["abc/perl-error-tail"]; + const RECOVERY_PROBES: &[&str] = &["abc/perl-error-tail", "halstead/perl-error-tail-keys"]; /// Deepest `tree_sitter` node depth reachable from the root. /// diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index 923bd8a7..d5cea396 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -156,6 +156,7 @@ Read it as follows. | `loc/wide-cfg-test-mod` | Rust | width | the `exclude_tests` prune arm's row bookkeeping (#1417) | linear | | `halstead/perl-dash-keys` | Perl | width | the dash-key workaround's `=>` lookup per key (#1545) | linear | | `abc/perl-bareword-keys` | Perl | width | the bareword-key workaround's `=>` lookup per key (#1572) | linear | +| `halstead/kotlin-short-interpolations` | Kotlin | width | the `$name` test's previous-token lookup in one string (#1584) | linear | | `halstead/kotlin-noinline-params` | Kotlin | width | the modifier recovery guard's next-token lookup (#1558) | linear | | `abc/perl-pattern-list` | Perl | width | ABC's pattern-role climb through `( … )` (#1467) | linear | | `cyclomatic/cpp-plus-chain` | C++ | depth | shape control for the three rows below | linear | @@ -165,7 +166,8 @@ Read it as follows. | `abc/perl-or-chain` | Perl | depth | the misparse check's spine climb (#1273) | linear | | `abc/perl-misparsed-comparison` | Perl | depth | the same climb on a chain the misparse absorbed (#1273) | linear | | `abc/perl-and-statements` | Perl | width | the misparse check's operator lookup in a block (#1273) | linear | -| `abc/perl-error-tail` | Perl | width | `perl_binding_rewrites` under a flat `ERROR` root (#1540) | linear | +| `abc/perl-error-tail` | Perl | width | the binding, bareword-key and misparse lookups under a flat `ERROR` root (#1540, #1584) | linear | +| `halstead/perl-error-tail-keys` | Perl | width | the key and sign lookups under the same `ERROR` (#1584) | linear | | `cognitive/bash-test-connective-run` | Bash | depth | the comparison-run end of a `[ … -o … ]` test (#1536) | linear | | `abc/bash-arith-assign-chain` | Bash | depth | ABC's `=` comparison test on `(( a = … ))` (#1536) | linear | | `abc/bash-eq-and-test` | Bash | depth | the same test on a `[[ … = … && … ]]` run (#1536) | linear | @@ -395,13 +397,28 @@ Removing only the kind gate in front of the Perl lookup moved nothing: `abc/perl-and-statements` held at 1.19. On a block that gate is not what keeps the walk linear, because the lookup seeks the child by byte. That seek is `O(log n)` only where the grammar stores the children as -a balanced repeat, though, and an `ERROR` that recovery built by -skipping tokens is one flat array. So `abc/perl-error-tail` carries -only `=~` in its body. With a bareword or a `&&` chain added it fits -2.09 on today's tree ([#1584][flat-error]). It is also the one probe -whose shape is meant not to parse: `RECOVERY_PROBES` in `shapes.rs` -exempts it from the parse check and asserts that its root is still an -`ERROR`. +a balanced repeat. An `ERROR` that recovery built by skipping tokens is +one flat array, which the seek scans from its start, and no tree-sitter +call reaches a child of one in better than its index +([#1584][flat-error]). So the metric walk and the `ops` walk record the +children of every node wider than 64 as they push them, in a +`SiblingCache` they lend through `Ancestors`, and +`Ancestors::siblings_after` / `siblings_before` binary-search that +array instead of seeking. `Ancestors::previous_sibling` reads it too: +it scanned its parent from the start, which made a valid Kotlin string +of `$name` interpolations quadratic (11.5 s for 8 000 names). With +`Ancestors::lookup` patched to ignore the cache, these three probes go +over bound and nothing else moves: + +| Walk put back | Probe that fails | Clean | Regressed | Top rung | +|---|---|---|---|---| +| sibling lookups seek in the flat `ERROR` | `abc/perl-error-tail` | 1.08 | 2.10 | 8.8 ms → 326 ms | +| (same patch) | `halstead/perl-error-tail-keys` | 1.04 | 1.74 | 14.5 ms → 96.0 ms | +| (same patch) | `halstead/kotlin-short-interpolations` | 0.96 | 2.02 | 1.1 ms → 589 ms | + +These two are the probes whose shape is meant not to parse: +`RECOVERY_PROBES` in `shapes.rs` exempts them from the parse check and +asserts that each root is still an `ERROR`. Treat the linear bounds above as covering the walk's ancestor *chain* threading, not every `O(depth)` lookup in the crate. diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index e958bbb9..d9d259e3 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -135,7 +135,13 @@ fn perl_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { P::And if !perl_and_is_operator(&op, node) => { perl_last_operand(node).map(|o| (o, false)) } - P::DASH if perl_dash_signs_keyword_key(&op, node) => { + // The `-`'s parent is all the chain the test reads. + P::DASH + if perl_dash_signs_keyword_key( + &op, + Ancestors::known(std::slice::from_ref(node)), + ) => + { wrapped_operand(node).map(|o| (o, false)) } P::BANG | P::Not => wrapped_operand(node).map(|o| (o, true)), @@ -280,11 +286,12 @@ fn perl_count_slot(slot: Option, conditions: &mut f64) { // not scanned child by child. Without them every spine node climbed to // the top and every chain scanned its statement list, both quadratic. // The `ERROR` is the one wide parent left, holding every chain of an -// unclosed list, so the operator before `child` is found by index rather -// than by a scan from the `ERROR`'s start. +// unclosed list. Recovery builds it as one flat array, which a seek by +// byte scans from its start, so the operator before `child` is read off +// the children the walk recorded (#1584). fn perl_misparse_absorbs_leading_operand<'a>( node: &Node<'a>, - mut ancestors: impl Iterator>, + ancestors: Ancestors<'a, '_>, ) -> bool { use Perl as P; @@ -293,18 +300,18 @@ fn perl_misparse_absorbs_leading_operand<'a>( { return false; } - let mut child = *node; - let spine_parent = ancestors.find(|parent| { + let (mut child, mut child_ancestors) = (*node, ancestors); + let spine_parent = ancestors.iter(node).find(|(parent, above)| { let on_spine = perl_leading_slot(parent).is_some_and(|lead| lead.id() == child.id()); if on_spine { - child = *parent; + (child, child_ancestors) = (*parent, *above); } !on_spine }); - spine_parent.is_some_and(|parent| { + spine_parent.is_some_and(|(parent, _)| { (parent.kind_id() == P::BinaryExpression as u16 || parent.is_error()) - && parent - .children_before(&child) + && child_ancestors + .siblings_before(&child) .find(|sibling| !sibling.as_tree_sitter().is_extra()) .is_some_and(|operator| { matches!( @@ -654,8 +661,7 @@ impl Abc for PerlCode { // condition (issue #403). Covers `&&`, `||`, `//`, // `and`, `or` — not `xor`, which is a value (#1536). P::AMPAMP | P::PIPEPIPE | P::SLASHSLASH | P::And | P::Or => { - let mut above = ancestors.iter(node).map(|(ancestor, _)| ancestor); - if let Some(chain) = above.next() { + if let Some((chain, above)) = ancestors.iter(node).next() { let absorbed = perl_misparse_absorbs_leading_operand(&chain, above); perl_count_chain_operands(&chain, absorbed, &mut stats.conditions); } @@ -696,8 +702,7 @@ impl Abc for PerlCode { // and adds the three operand slots (issue #1102). P::TernaryExpression => { stats.conditions += 1.; - let above = ancestors.iter(node).map(|(ancestor, _)| ancestor); - let absorbed = perl_misparse_absorbs_leading_operand(node, above); + let absorbed = perl_misparse_absorbs_leading_operand(node, ancestors); perl_walk_ternary(node, absorbed, &mut stats.conditions); } // Phase-2B (issue #403): condition slots, each read by the diff --git a/src/metrics/perl_keyword_key_tests.rs b/src/metrics/perl_keyword_key_tests.rs index 5b1dfed9..597f34e5 100644 --- a/src/metrics/perl_keyword_key_tests.rs +++ b/src/metrics/perl_keyword_key_tests.rs @@ -888,3 +888,62 @@ fn perl_bareword_outside_a_key_keeps_its_call() { ); } } + +/// A file whose tail fails to parse leaves every key, chain and sign in +/// it a direct child of one flat `ERROR` root, and the key and misparse +/// lookups ask for a sibling in that `ERROR` once per child. Both walks +/// lend the `SiblingCache` that answers them from the children they +/// recorded rather than by a scan from the `ERROR`'s start (#1584); a +/// walk that stopped recording trips the debug assertion in the lookup +/// here, since the answers themselves would not change. Every value is +/// `main`'s, measured before #1584: twelve `last` calls, the `name` +/// keys none, and `-a` / `-and` billed whole. +#[cfg(feature = "perl")] +#[test] +fn perl_flat_error_reads_siblings_from_the_walk() { + // Past `WIDE_PARENT` in `big-code-analysis-ast/src/node/ancestors.rs`, + // the width from which a walk records a node's children. + const RECORDED_WIDTH: usize = 64; + let body = "{\nlast,\n$a && $b,\n- and,\n-a => 1,\nname => 1,\n$c > $d && $e,\n"; + let source = format!("my @r = ({}", body.repeat(12)); + let ast = Ast::parse(Source::new(LANG::Perl, source.as_bytes())).expect("perl is enabled"); + let root = ast.as_tree_sitter().root_node(); + assert!( + root.is_error() && root.child_count() > RECORDED_WIDTH, + "the fixture must parse to a wide flat ERROR root" + ); + + let space = ast + .metrics(MetricsOptions::default()) + .expect("metrics yields a root space"); + let (abc, halstead) = (&space.metrics.abc, &space.metrics.halstead); + assert_eq!( + ( + abc.branches_sum(), + abc.conditions_sum(), + abc.assignments_sum() + ), + (12, 48, 1) + ); + assert_eq!( + [ + halstead.unique_operators(), + halstead.total_operators(), + halstead.unique_operands(), + halstead.total_operands(), + ], + [9, 207, 11, 133] + ); + + let ops = ast.ops().expect("ops yields a root space"); + assert_eq!( + ops.operands, + [ + "$a", "$b", "$c", "$d", "$e", "-a", "-and", "1", "@r", "last", "name" + ] + ); + assert_eq!( + ops.operators, + ["$", "&&", "()", ",", "=", "=>", ">", "my", "{}"] + ); +} diff --git a/src/ops.rs b/src/ops.rs index c3385e1b..9a6dac6e 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -11,7 +11,7 @@ use std::borrow::Cow; use crate::checker::Checker; use crate::error::MetricsError; use crate::getter::Getter; -use crate::node::{Ancestors, Node}; +use crate::node::{Ancestors, Node, SiblingCache}; use crate::spaces::{SpaceKind, line_span, push_children}; use crate::halstead::{Halstead, HalsteadMaps}; @@ -310,6 +310,8 @@ pub(crate) fn ops_inner( // maintained by the same truncate/push rule as // `spaces::compute::metrics_inner` (#1084). let mut chain: Vec> = Vec::new(); + // As in `metrics_inner`: the children of each wide node on `chain`. + let mut siblings = SiblingCache::default(); let mut state_stack: Vec = Vec::new(); let mut last_level = 0; @@ -330,7 +332,7 @@ pub(crate) fn ops_inner( last_level = level; } - let ancestors = Ancestors::checked(&chain, &node); + let ancestors = Ancestors::checked(&chain, &node).with_siblings(&siblings); // Same predicate `spaces::compute::metrics_inner` opens on, so // the two walks agree on which nodes become spaces. The @@ -366,9 +368,7 @@ pub(crate) fn ops_inner( // mirrors (which differ by `State` payload) it is reused directly // rather than duplicated. The source-order-then-reverse ordering // it encapsulates is load-bearing for suppression attribution. - // The returned child slice is only useful to `metrics_inner`, - // which seeds their cognitive nesting; `ops` just walks them. - push_children( + let pushed = push_children( &mut cursor, &node, Walk { @@ -377,6 +377,7 @@ pub(crate) fn ops_inner( }, &mut stack, ); + siblings.record(&node, depth, pushed.iter().rev().map(|(child, _)| *child)); } finalize::(&mut state_stack, usize::MAX); diff --git a/src/spaces.rs b/src/spaces.rs index 7671ca52..7385b468 100644 --- a/src/spaces.rs +++ b/src/spaces.rs @@ -68,7 +68,7 @@ use crate::preproc::PreprocResults; use crate::checker::Checker; use crate::error::MetricsError; -use crate::node::{Ancestors, Cursor, Node}; +use crate::node::{Ancestors, Cursor, Node, SiblingCache}; use crate::suppression::{ Suppression, SuppressionKind, SuppressionScope, parse_marker as parse_suppression_marker, }; diff --git a/src/spaces/compute.rs b/src/spaces/compute.rs index 2b07944b..fff1405e 100644 --- a/src/spaces/compute.rs +++ b/src/spaces/compute.rs @@ -755,6 +755,9 @@ pub(crate) fn metrics_inner( // index instead of through `Node::parent`, which `tree_sitter` // resolves by descending from the root (#1084). let mut chain: Vec> = Vec::new(); + // The children of each wide node on `chain`, so a predicate asking + // for a sibling in a flat `ERROR` does not scan it (#1584). + let mut siblings = SiblingCache::default(); let mut state_stack: Vec = Vec::new(); let mut last_level = 0; // Per-node cognitive nesting, inherited down the walk. Deliberately @@ -827,7 +830,7 @@ pub(crate) fn metrics_inner( // and `chain` is already correct here — the truncate above is // the only thing that touches it between the pop and this // point (#1100). - let ancestors = Ancestors::checked(&chain, &node); + let ancestors = Ancestors::checked(&chain, &node).with_siblings(&siblings); // Prune test-only subtrees before any per-metric work runs. // The hook is gated on `exclude_tests` so the default @@ -916,6 +919,7 @@ pub(crate) fn metrics_inner( }, &mut stack, ); + siblings.record(&node, depth, pushed.iter().rev().map(|(child, _)| *child)); if selected.contains(Metric::Cognitive) { propagate_nesting_to_children(&node, pushed, &mut nesting_map); From 5f97527a423273f14a0be76c1bb619467b9628f8 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:12:04 -0700 Subject: [PATCH 17/34] fix(perl): report each call once in find and count tree-sitter-perl parses a call's name as a bareword call nested in the call, and an auto-quoted hash key as a bareword call too, so `bca find --type call` reported `foo(1)` twice and every `name =>` and `$h{-name}` key once. ABC already skipped both through a gate that needs the ancestor chain, which the byte-less `is_call` lacks. The gate moves to lang_helpers::perl::perl_bareword_is_call, which ABC and a new Checker::is_call_with_code share; the `call` filter now calls the chain-carrying spelling, whose default forwards to `is_call`. The bareword is kept, gated, rather than dropped: it is the only node for `foo;`, `&foo;` and the wrapper-less `foo(1` of error recovery. Perl's `call` set now equals its ABC branches. The key test reads the token after each bareword, a seek that is quadratic under a flat ERROR (#1584), so act_on_node now records and lends a SiblingCache as the metric and ops walks do. The new count/perl-error-tail-calls probe fits 0.88 with it and 1.89 with the cache left unlent. Fixes #1580 --- .bca-baseline.toml | 2 +- big-code-analysis-ast/src/checker.rs | 33 +++-- big-code-analysis-ast/src/checker/perl.rs | 7 + .../src/lang_helpers/perl.rs | 27 ++++ big-code-analysis-ast/src/node.rs | 45 +++++- big-code-analysis-ast/src/parser.rs | 18 ++- big-code-analysis-bench/src/shapes.rs | 57 +++++++- big-code-analysis-book/src/commands/nodes.md | 4 + big-code-analysis-book/src/metrics.md | 2 +- docs/development/benchmarking.md | 18 ++- fuzz/src/lib.rs | 13 +- src/metrics/abc/perl.rs | 15 +- src/metrics/perl_keyword_key_tests.rs | 138 +++++++++++++++++- 13 files changed, 327 insertions(+), 52 deletions(-) diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 4cf68e45..9945e733 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -279,7 +279,7 @@ value = 42.0 path = "big-code-analysis-ast/src/parser.rs" qualified = "Parser::filters" metric = "halstead.effort" -value = 87390.78947368421 +value = 90363.0 [[entry]] path = "big-code-analysis-ast/src/preproc.rs" diff --git a/big-code-analysis-ast/src/checker.rs b/big-code-analysis-ast/src/checker.rs index 2a1044ea..1b747b01 100644 --- a/big-code-analysis-ast/src/checker.rs +++ b/big-code-analysis-ast/src/checker.rs @@ -475,7 +475,7 @@ pub trait Checker { /// TypeScript, TSX, PHP, Ruby (`super` / `yield` count as branches) /// and Rust (`?` does). /// - /// The relation is not a plain superset everywhere. Four languages + /// The relation is not a plain superset everywhere. Three languages /// depart from it, each in its own direction: /// /// * Tcl and iRules bill a mutator command (`incr`, `append`, @@ -487,17 +487,15 @@ pub trait Checker { /// * Elixir excludes its definition and directive `Call`s (`def`, /// `defmodule`, `alias`, `import`, …) from `branches` while /// counting each `|>` pipeline step. - /// * Perl counts a bareword call only at the outermost dispatch - /// site: in `print shift;` the wrapper - /// `call_expression_with_spaced_args` is the branch and the - /// `print` bareword it holds is skipped, so that bareword is an - /// `is_call` match with no branch behind it. So is an auto-quoted - /// hash key (`(name => 1)`, `$h{-name}`), which the grammar parses - /// as a bareword call and ABC reads as the string it is. /// - /// The two sets coincide only in C, Objective-C, Go, Python, Lua - /// and Bash, which have neither a separate construction syntax nor - /// a gated branch arm. + /// The two sets coincide in C, Objective-C, Go, Python, Lua and + /// Bash, which have neither a separate construction syntax nor a + /// gated branch arm, and in Perl, whose one gated arm ABC shares + /// with [`is_call_with_code`](Self::is_call_with_code): a bareword + /// is no call of its own as a wrapper's callee (`print` in + /// `print shift;`) or as an auto-quoted key (`(name => 1)`). That + /// gate needs the ancestor chain, so for Perl this byte-less form is + /// only the kind table the override narrows. // `is_call` and each language's ABC branch arm classify the same // nodes through parallel `matches!()` tables, so they drift // silently; the divergence is pinned by @@ -508,6 +506,19 @@ pub trait Checker { fn is_call(_: &Node) -> bool { false } + /// [`is_call`](Self::is_call), given the source bytes and the + /// chain the caller descended through, which the `call` filter + /// hands it. The default forwards to [`is_call`](Self::is_call). + /// + /// Perl overrides it: the grammar nests a call's name inside the + /// call as a bareword call of its own, and parses an auto-quoted + /// hash key as one, so only the position says which words are + /// calls (#1580). + #[inline] + #[must_use] + fn is_call_with_code<'a>(node: &Node<'a>, _code: &[u8], _ancestors: Ancestors<'a, '_>) -> bool { + Self::is_call(node) + } /// Whether `node`, met among a parameter list's children, is /// punctuation or a keyword rather than a parameter (`nargs` skips it). #[inline] diff --git a/big-code-analysis-ast/src/checker/perl.rs b/big-code-analysis-ast/src/checker/perl.rs index cb2fb89f..7e15efed 100644 --- a/big-code-analysis-ast/src/checker/perl.rs +++ b/big-code-analysis-ast/src/checker/perl.rs @@ -2,6 +2,7 @@ #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] use super::*; +use crate::lang_helpers::perl::perl_bareword_is_call; impl Checker for PerlCode { fn is_comment(node: &Node) -> bool { @@ -42,6 +43,12 @@ impl Checker for PerlCode { ) } + fn is_call_with_code<'a>(node: &Node<'a>, _code: &[u8], ancestors: Ancestors<'a, '_>) -> bool { + Self::is_call(node) + && (node.kind_id() != Perl::CallExpressionWithBareword as u16 + || perl_bareword_is_call(node, ancestors)) + } + // `NormalComma` (369) is the separator inside a subroutine signature; // the bare `COMMA` (12) it wraps is that node's *child*, so listing // only `COMMA` let every signature separator count as a parameter diff --git a/big-code-analysis-ast/src/lang_helpers/perl.rs b/big-code-analysis-ast/src/lang_helpers/perl.rs index 9b90f7e4..e40ffadf 100644 --- a/big-code-analysis-ast/src/lang_helpers/perl.rs +++ b/big-code-analysis-ast/src/lang_helpers/perl.rs @@ -188,6 +188,33 @@ pub fn perl_bareword_is_key<'a>(word: &Node<'a>, ancestors: Ancestors<'a, '_>) - } } +/// Whether `word`, a Perl `call_expression_with_bareword` whose ancestor +/// chain is `ancestors`, is a call site of its own. +/// +/// The grammar nests the callee of every call that takes arguments as a +/// bareword call: `foo(1)`, `foo 1` and `sort { … } @x` each wrap a +/// `call_expression_with_bareword` spelling `foo` or `sort`, so the +/// wrapper is the call and the word inside it is its name. The word is +/// kept for the spellings with no wrapper — `foo;`, `&foo;`, and the +/// `foo(1` that error recovery leaves without one — so the wrapper +/// cannot be dropped instead. Nor is an auto-quoted key a call +/// ([`perl_bareword_is_key`]). ABC's branches and +/// `Checker::is_call_with_code` both ask this, so the two count the same +/// calls. +#[must_use] +pub fn perl_bareword_is_call<'a>(word: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + !ancestors.parent(word).is_some_and(|parent| { + matches!( + parent.kind_id().into(), + Perl::CallExpressionWithSpacedArgs + | Perl::CallExpressionWithSub + | Perl::CallExpressionWithArgsWithBrackets + | Perl::CallExpressionWithVariable + | Perl::CallExpressionRecursive + ) + }) && !perl_bareword_is_key(word, ancestors) +} + /// Whether `dash`, a Perl `-` token whose ancestor chain is /// `ancestors`, is the sign of an auto-quoted key rather than a unary /// minus: a `-and` / `-not` key ([`perl_dash_signs_keyword_key`]) or a diff --git a/big-code-analysis-ast/src/node.rs b/big-code-analysis-ast/src/node.rs index 3cd4c3b5..877451df 100644 --- a/big-code-analysis-ast/src/node.rs +++ b/big-code-analysis-ast/src/node.rs @@ -912,12 +912,25 @@ impl<'a> Search<'a> for Node<'a> { let mut chain: Vec> = std::iter::successors(self.parent(), Node::parent).collect(); chain.reverse(); let depth = chain.len(); + // As in the metric walk: the children of each wide node on + // `chain`, so a sibling lookup in a flat `ERROR` is a binary + // search rather than a seek from its start (#1584). Perl's + // `call` filter reads the token after every bareword (#1580). + // The seeded ancestors are recorded too, since the walk never + // pushes them. + let mut siblings = SiblingCache::default(); + for (at, ancestor) in chain.iter().enumerate() { + siblings.record(ancestor, at, ancestor.children()); + } stack.push((*self, depth)); while let Some((node, depth)) = stack.pop() { chain.truncate(depth); - action(&node, Ancestors::checked(&chain, &node)); + action( + &node, + Ancestors::checked(&chain, &node).with_siblings(&siblings), + ); chain.push(node); // Source order in, tail reversed in place, so the LIFO // `stack` yields the leftmost child first — pre-order with @@ -927,6 +940,11 @@ impl<'a> Search<'a> for Node<'a> { node.children_with(&mut cursor) .map(|child| (child, depth + 1)), ); + siblings.record( + &node, + depth, + stack[first_child..].iter().map(|(child, _)| *child), + ); stack[first_child..].reverse(); } } @@ -1860,6 +1878,31 @@ mod tests { assert!(visited > 3, "subtree is too small to prove much"); } + /// A walk started below the root records its seeded ancestors' + /// children too, so a sibling lookup off a wide one is answered from + /// the cache. Without them the lookup's debug assertion fails: the + /// cache is lent, and the wide parent on the chain was never pushed + /// by this walk (#1580). + #[cfg(feature = "mozjs")] + #[test] + fn act_on_node_records_a_wide_seeded_ancestor() { + // Past `WIDE_PARENT` in `node/ancestors.rs`. + const RECORDED_WIDTH: usize = 64; + let code = "var a = 1;\n".repeat(RECORDED_WIDTH + 6); + let tree = Tree::new::(code.as_bytes()); + let root = tree.get_root(); + assert!(root.child_count() > RECORDED_WIDTH, "the root must be wide"); + let subtree = root.child(3).expect("fixture has a fourth statement"); + + let mut after = None; + subtree.act_on_node(&mut |node, ancestors| { + if node.id() == subtree.id() { + after = Some(ancestors.siblings_after(node).count()); + } + }); + assert_eq!(after, Some(root.child_count() - 4)); + } + /// `previous_sibling` must not answer "no previous sibling" when the /// chain it was handed belongs to a different node. /// diff --git a/big-code-analysis-ast/src/parser.rs b/big-code-analysis-ast/src/parser.rs index d5e30b99..7a936653 100644 --- a/big-code-analysis-ast/src/parser.rs +++ b/big-code-analysis-ast/src/parser.rs @@ -133,7 +133,7 @@ impl ParserTrait for P } fn filters(&self, requested: &[String]) -> Filter<'_> { - // Borrowed by the `"function"` and `"string"` arms below, which + // Borrowed by the `"call"`, `"function"` and `"string"` arms below, which // is why `Filter` carries a lifetime. let code = self.code(); let mut res: Vec>> = Vec::new(); @@ -141,13 +141,15 @@ impl ParserTrait for P let f = f.as_str(); match f { "all" => res.push(Box::new(|_: &Node, _| -> bool { true })), - // `is_call` / `is_comment` / `is_error` take `&Node` and - // nothing else, so no language *can* make them - // text-dependent. The #1162 gap is confined by - // construction to the `Checker` predicates that accept - // `code`, and `"function"` and `"string"` are the two - // filters that reach one. - "call" => res.push(Box::new(|node: &Node, _| T::is_call(node))), + // `is_call_with_code`, not `is_call`: Perl parses a + // call's name and an auto-quoted hash key as bareword + // calls, and only the chain tells those from a call of + // their own (#1580). `is_comment` / `is_error` take + // `&Node` and nothing else, so no language *can* make + // them text-dependent. + "call" => res.push(Box::new(move |node: &Node, ancestors| { + T::is_call_with_code(node, code, ancestors) + })), "comment" => res.push(Box::new(|node: &Node, _| T::is_comment(node))), "error" => res.push(Box::new(|node: &Node, _| T::is_error(node))), // `is_string_with_code`, not `is_string`: a Tcl-family diff --git a/big-code-analysis-bench/src/shapes.rs b/big-code-analysis-bench/src/shapes.rs index f75a23c6..ebdbe7e7 100644 --- a/big-code-analysis-bench/src/shapes.rs +++ b/big-code-analysis-bench/src/shapes.rs @@ -640,6 +640,17 @@ pub fn perl_error_tail_keys(width: usize) -> String { format!("my @r = ({}", "{\n- and,\nnot,\n-a => 1,\n".repeat(width)) } +/// Perl: `my @r = (` followed by `width` copies of +/// `{⏎last,⏎name => 1,⏎`, never closed. +/// +/// [`perl_error_tail`]'s flat `ERROR`, holding a bareword call and a +/// bareword key the grammar also reads as a call: the `call` filter +/// asks of each whether a `=>` follows it. +#[must_use] +pub fn perl_error_tail_calls(width: usize) -> String { + format!("my @r = ({}", "{\nlast,\nname => 1,\n".repeat(width)) +} + /// Bash: `if [ "$a" = 0 -o "$a" = 0 -a … ]; then :; fi`, the /// connectives alternating. /// @@ -708,9 +719,10 @@ pub fn cpp_plus_chain(depth: usize) -> String { /// The walk a probe times. /// -/// Two seams reach the AST walker — `Ast::metrics` and `Ast::ops` — and -/// they share no options type, so the selection a metric probe needs -/// lives on the variant that uses it rather than on every probe (#1110). +/// Three seams reach an AST walk — `Ast::metrics`, `Ast::ops` and the +/// `Ast::count` walk `bca find` shares — and they share no options type, +/// so the selection a metric probe needs lives on the variant that uses +/// it rather than on every probe (#1110). #[derive(Clone, Copy)] pub enum Workload { /// `Ast::metrics` under `selection`. @@ -736,12 +748,18 @@ pub enum Workload { /// Headline value the walk produces on the probe's shape. reading: fn(&Ops) -> u64, }, + /// `Ast::count` under one `--type` filter, the walk `bca count` and + /// `bca find` share. Its reading is the matching node count. + Count { + /// The filter, as `bca count --type` spells it. + filter: &'static str, + }, } impl Workload { /// The options this workload walks under, resolved once so option - /// resolution stays outside the timed region. `Ast::ops` takes none, - /// so the default stands in and is ignored. + /// resolution stays outside the timed region. `Ast::ops` and + /// `Ast::count` take none, so the default stands in and is ignored. #[must_use] pub fn options(self) -> MetricsOptions { match self { @@ -752,7 +770,7 @@ impl Workload { } => MetricsOptions::default() .with_only(selection) .with_exclude_tests(exclude_tests), - Self::Ops { .. } => MetricsOptions::default(), + Self::Ops { .. } | Self::Count { .. } => MetricsOptions::default(), } } @@ -770,6 +788,10 @@ impl Workload { Ok(match self { Self::Metrics { reading, .. } => reading(&black_box(ast.metrics(options)?).metrics), Self::Ops { reading } => reading(&black_box(ast.ops()?)), + Self::Count { filter } => { + let (matching, _) = black_box(ast.count(&[filter.to_owned()])); + u64::try_from(matching).unwrap_or(u64::MAX) + } }) } } @@ -1666,6 +1688,23 @@ pub const PROBES: &[Probe] = &[ quadratic in the tail. The reading is three operands \ per copy of the body.", }, + Probe { + name: "count/perl-error-tail-calls", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Count { filter: "call" }, + render: perl_error_tail_calls, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "`abc/perl-error-tail`'s flat `ERROR` under the walk \ + `bca count` and `bca find` share. The `call` filter \ + asks the bareword-key test of every bareword (#1580), \ + which reads the token after it; that walk lends no \ + `SiblingCache` unless `act_on_node` records one, and \ + a seek from the `ERROR`'s start is quadratic in the \ + tail. The reading is one call, `last`, per copy of \ + the body.", + }, Probe { name: "halstead/kotlin-short-interpolations", lang: LANG::Kotlin, @@ -1885,7 +1924,11 @@ mod tests { /// Probes whose shape deliberately fails to parse; see /// [`shapes_parse_without_error`]. - const RECOVERY_PROBES: &[&str] = &["abc/perl-error-tail", "halstead/perl-error-tail-keys"]; + const RECOVERY_PROBES: &[&str] = &[ + "abc/perl-error-tail", + "halstead/perl-error-tail-keys", + "count/perl-error-tail-calls", + ]; /// Deepest `tree_sitter` node depth reachable from the root. /// diff --git a/big-code-analysis-book/src/commands/nodes.md b/big-code-analysis-book/src/commands/nodes.md index 0117e4fb..bc25ad25 100644 --- a/big-code-analysis-book/src/commands/nodes.md +++ b/big-code-analysis-book/src/commands/nodes.md @@ -60,6 +60,10 @@ with a literal `call` node cannot be matched by name — use its numeric in C#, Java, Groovy, Kotlin, C++, JavaScript, TypeScript, PHP, Ruby and Rust; see [Where a constructor call lands](../metrics.md#abc-constructor-attribution). + Each call matches once. In Perl, where the grammar parses a call's + name as a call of its own, `foo(1)` matches as the whole call only, + and an auto-quoted hash key (`(name => 1)`, `$h{-name}`), which it + parses as a call too, does not match at all. - `comment` — a comment of any of the language's forms. - `string` — a string *literal*. Type-annotation keywords that share the literal's node name (TypeScript's and PHP's `: string`) are excluded. diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 7111a61b..e99b6b9b 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -163,7 +163,7 @@ application would over-count. | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | | C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | | Perl | A comparison's right operand is not a condition, although the grammar parses it as one | tree-sitter-perl 1.1.2 lets a chaining comparison (`<`, `>`, `<=`, `>=`, `lt`, `gt`, `le`, `ge`, `==`, `!=`, `eq`, `ne`) take everything of lower precedence to its right as its operand, so `$a > $b ? 1 : 2` parses as `$a > ($b ? 1 : 2)` and `$a > $b && $c` as `$a > ($b && $c)`. The ternary's condition slot and the chain's first operand then held `$b`, which scored as a condition of its own: one above the JavaScript twin for each misparsed comparison. The walker recognises the shape and leaves that operand to the comparison, so both spellings score what the parenthesised `($a > $b) ? 1 : 2` and `($a > $b) && $c` do. An explicit `$a > ($b ? 1 : 2)` is unaffected. `<=>`, `cmp`, `=~` and `!~` do not chain and parse correctly. The workaround goes when the grammar is fixed upstream (#1273). | -| Perl | An auto-quoted hash key is a string, not a call | Perl quotes a bareword before `=>` and alone in a hash subscript, a leading `-` included, so `(name => 1)`, `new(name => $x)`, `(-name => 1)` and `$h{-name}` hold the strings `"name"` and `"-name"`. tree-sitter-perl 1.1.2 parses each word as a call, which scored a branch, and each `-name` as a negation, which billed a Halstead `-` operator and the operand `name`. Each now scores as its quoted twin `('name' => 1)` does, and Halstead bills the key as one operand spelled as written: `name`, `-name`. A package-qualified word (`Foo::Bar => 1`), which `=>` does not quote, a word with arguments (`name() => 1`) and a bareword value (`(a => foo)`) keep their call. The workaround goes when the grammar is fixed upstream (#1571, #1572). | +| Perl | An auto-quoted hash key is a string, not a call | Perl quotes a bareword before `=>` and alone in a hash subscript, a leading `-` included, so `(name => 1)`, `new(name => $x)`, `(-name => 1)` and `$h{-name}` hold the strings `"name"` and `"-name"`. tree-sitter-perl 1.1.2 parses each word as a call, which scored a branch, and each `-name` as a negation, which billed a Halstead `-` operator and the operand `name`. Each now scores as its quoted twin `('name' => 1)` does, and Halstead bills the key as one operand spelled as written: `name`, `-name`. A package-qualified word (`Foo::Bar => 1`), which `=>` does not quote, a word with arguments (`name() => 1`) and a bareword value (`(a => foo)`) keep their call. `bca find --type call` reports the same calls ABC counts (#1580). The workaround goes when the grammar is fixed upstream (#1571, #1572). | #### Where a constructor call lands {#abc-constructor-attribution} diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index d5cea396..e5a89119 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -58,8 +58,9 @@ git submodule update --init --recursive ## The complexity-class gate {#complexity-gate} Each probe pairs a generated source shape with the workload that -exercises one hot path — a metric selection through `Ast::metrics`, or -the operator/operand walk behind `Ast::ops`. The shape is rendered at three doubling +exercises one hot path — a metric selection through `Ast::metrics`, +the operator/operand walk behind `Ast::ops`, or a `--type` filter +through `Ast::count`. The shape is rendered at three doubling sizes, every (probe, size) cell is measured once per round with the visit order rotated between rounds, and the reported figure is the slope of `ln(time)` against `ln(size)`. A linear walk sits near 1.0; @@ -168,6 +169,7 @@ Read it as follows. | `abc/perl-and-statements` | Perl | width | the misparse check's operator lookup in a block (#1273) | linear | | `abc/perl-error-tail` | Perl | width | the binding, bareword-key and misparse lookups under a flat `ERROR` root (#1540, #1584) | linear | | `halstead/perl-error-tail-keys` | Perl | width | the key and sign lookups under the same `ERROR` (#1584) | linear | +| `count/perl-error-tail-calls` | Perl | width | the `call` filter's bareword-key lookup under the same `ERROR`, on the `find` / `count` walk (#1580) | linear | | `cognitive/bash-test-connective-run` | Bash | depth | the comparison-run end of a `[ … -o … ]` test (#1536) | linear | | `abc/bash-arith-assign-chain` | Bash | depth | ABC's `=` comparison test on `(( a = … ))` (#1536) | linear | | `abc/bash-eq-and-test` | Bash | depth | the same test on a `[[ … = … && … ]]` run (#1536) | linear | @@ -416,7 +418,16 @@ over bound and nothing else moves: | (same patch) | `halstead/perl-error-tail-keys` | 1.04 | 1.74 | 14.5 ms → 96.0 ms | | (same patch) | `halstead/kotlin-short-interpolations` | 0.96 | 2.02 | 1.1 ms → 589 ms | -These two are the probes whose shape is meant not to parse: +The walk `bca find` and `bca count` share, `act_on_node`, keeps the same +chain and lent no cache until Perl's `call` filter started asking the +bareword-key test of every bareword ([#1580][find-call]). It records +one now, and `count/perl-error-tail-calls` prices it: + +| Walk put back | Probe that fails | Clean | Regressed | Top rung | +|---|---|---|---|---| +| `act_on_node` lends no cache | `count/perl-error-tail-calls` | 0.88 | 1.89 | 2.8 ms → 84.5 ms | + +These three are the probes whose shape is meant not to parse: `RECOVERY_PROBES` in `shapes.rs` exempts them from the parse check and asserts that each root is still an `ERROR`. @@ -518,6 +529,7 @@ a walk's chain bookkeeping, not just around a change to its cost. The [space-merge]: https://github.com/dekobon/big-code-analysis/issues/1106 [perl-bash-probes]: https://github.com/dekobon/big-code-analysis/issues/1575 [flat-error]: https://github.com/dekobon/big-code-analysis/issues/1584 +[find-call]: https://github.com/dekobon/big-code-analysis/issues/1580 The eleven control probes are what make the other readings mean something. diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index e0c9d6a8..1271d70f 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -53,16 +53,17 @@ pub mod nested; /// Node-kind filters handed to [`Ast::count`] and [`Ast::find`]. /// -/// `"function"` and `"string"` earn their places twice over: they are the -/// two filters that reach a `Checker` predicate taking `code`, and both -/// consult the ancestor chain `find` and `count` thread (#1381) — the -/// Tcl-family `"string"` on every braced word — so they are the filters -/// a deeply-nested or wide input exercises hardest. +/// `"call"`, `"function"` and `"string"` earn their places twice over: +/// they are the filters that reach a `Checker` predicate taking `code`, +/// and each consults the ancestor chain `find` and `count` thread +/// (#1381, #1580) — the Tcl-family `"string"` on every braced word, the +/// Perl `"call"` on every bareword — so they are the filters a +/// deeply-nested or wide input exercises hardest. /// /// **The order is load-bearing, and `"all"` must stay last.** /// `Filter::any` returns on its first matching predicate, and `"all"` is /// `|_| true`, so listing it first makes every other entry unreachable — -/// `is_call`, `is_comment`, `is_error`, `is_string_with_code` and +/// `is_call_with_code`, `is_comment`, `is_error`, `is_string_with_code` and /// `is_func_with_code` are then never called on any node, in any target. /// It was first here until a review caught it, which had quietly reduced /// `count` and `find` to a bare walk and left the `"function"` predicate diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index d9d259e3..84ab0db8 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -14,7 +14,7 @@ use super::{ wrapped_operand, }; use crate::lang_helpers::perl::{ - perl_and_is_operator, perl_bareword_is_key, perl_dash_signs_keyword_key, perl_not_is_key, + perl_and_is_operator, perl_bareword_is_call, perl_dash_signs_keyword_key, perl_not_is_key, }; use crate::*; @@ -578,18 +578,7 @@ impl Abc for PerlCode { // `(name => 1)`, `(-name => 1)` and `$h{-name}` as calls of // `name`, and `$h{-foo}` as a file test on a bareword `oo`, // where Perl holds a string (#1545, #1572). - P::CallExpressionWithBareword - if !ancestors.parent(node).is_some_and(|p| { - matches!( - p.kind_id().into(), - P::CallExpressionWithSpacedArgs - | P::CallExpressionWithSub - | P::CallExpressionWithArgsWithBrackets - | P::CallExpressionWithVariable - | P::CallExpressionRecursive - ) - }) && !perl_bareword_is_key(node, ancestors) => - { + P::CallExpressionWithBareword if perl_bareword_is_call(node, ancestors) => { stats.branches += 1.; } // Numeric, string, and pattern-match comparison operators diff --git a/src/metrics/perl_keyword_key_tests.rs b/src/metrics/perl_keyword_key_tests.rs index 597f34e5..c5cb9577 100644 --- a/src/metrics/perl_keyword_key_tests.rs +++ b/src/metrics/perl_keyword_key_tests.rs @@ -897,7 +897,8 @@ fn perl_bareword_outside_a_key_keeps_its_call() { /// walk that stopped recording trips the debug assertion in the lookup /// here, since the answers themselves would not change. Every value is /// `main`'s, measured before #1584: twelve `last` calls, the `name` -/// keys none, and `-a` / `-and` billed whole. +/// keys none, and `-a` / `-and` billed whole. `find` and `count` walk +/// with `act_on_node`, which lends the same cache. #[cfg(feature = "perl")] #[test] fn perl_flat_error_reads_siblings_from_the_walk() { @@ -946,4 +947,139 @@ fn perl_flat_error_reads_siblings_from_the_walk() { ops.operators, ["$", "&&", "()", ",", "=", "=>", ">", "my", "{}"] ); + + // The `call` filter of `find` and `count` asks the same key test of + // every bareword, through the walk `act_on_node` keeps (#1580): the + // twelve `last` calls, as ABC counts them, and no `name` key. + assert_eq!(ast.count(&["call".to_owned()]).0, 12); +} + +/// What `bca find --type call` reports on `source`: each node's kind and +/// text, in source order, and the root space's ABC branches. `is_call` +/// and ABC's branch arm share `perl_bareword_is_call`, so the two must +/// count the same calls (grammar-dispatch §7) — Perl has no other kind +/// of ABC branch. +#[cfg(feature = "perl")] +fn perl_calls(source: &str) -> (Vec<(String, String)>, usize) { + let ast = Ast::parse(Source::new(LANG::Perl, source.as_bytes())).expect("perl is enabled"); + let found = ast + .find(&["call".to_owned()]) + .expect("find is infallible") + .iter() + .map(|node| { + ( + node.kind().to_owned(), + source[node.start_byte()..node.end_byte()].to_owned(), + ) + }) + .collect::>(); + assert_eq!( + ast.count(&["call".to_owned()]).0, + found.len(), + "`count` and `find` disagree on `{source}`" + ); + let branches = ast + .metrics(MetricsOptions::default()) + .expect("metrics yields a root space") + .metrics + .abc + .branches_sum(); + let branches = usize::try_from(branches).expect("a fixture's branch count fits a usize"); + (found, branches) +} + +/// #1580's fixture: three auto-quoted keys the grammar parses as +/// bareword calls, and one call whose name it parses as a second. Before +/// the fix `find` reported all five; `foo(1)` is the only call. +#[cfg(feature = "perl")] +#[test] +fn perl_find_call_reports_one_node_per_call_1580() { + let source = "sub d { my %h = (name => 1, -size => 2); my $y = $h{-name}; return foo(1); }\n"; + let (found, branches) = perl_calls(source); + assert_eq!( + found, + [( + "call_expression_with_args_with_brackets".to_owned(), + "foo(1)".to_owned() + )] + ); + assert_eq!(branches, 1); +} + +/// Each call spelling is one call, reported on the node present for +/// that spelling: the wrapper where the grammar builds one around the +/// name, else the bareword. `foo(1` is the error-recovery shape with no +/// wrapper at all, so its bareword is the call; dropping bareword calls +/// instead of gating them would lose it, and `foo;` and `&foo;` with it +/// (grammar-dispatch §6). +#[cfg(feature = "perl")] +#[test] +fn perl_each_call_spelling_is_one_call_1580() { + const BRACKETS: &str = "call_expression_with_args_with_brackets"; + const SPACED: &str = "call_expression_with_spaced_args"; + const BAREWORD: &str = "call_expression_with_bareword"; + const METHOD: &str = "method_invocation"; + let rows: [(&str, &[(&str, &str)]); 14] = [ + ("foo(1);", &[(BRACKETS, "foo(1)")]), + ("foo();", &[(BRACKETS, "foo()")]), + ("&foo(1);", &[(BRACKETS, "&foo(1)")]), + ("Foo::bar(1);", &[(BRACKETS, "Foo::bar(1)")]), + ("foo 1;", &[(SPACED, "foo 1")]), + ("print \"x\";", &[(SPACED, "print \"x\"")]), + ( + "sort { $a <=> $b } @x;", + &[(SPACED, "sort { $a <=> $b } @x")], + ), + ("$obj->m(1);", &[(METHOD, "$obj->m(1)")]), + ( + "&$code(1);", + &[("call_expression_with_variable", "&$code(1)")], + ), + ( + "__SUB__->(1);", + &[("call_expression_recursive", "__SUB__->(1)")], + ), + ("foo;", &[(BAREWORD, "foo")]), + ("&foo;", &[(BAREWORD, "&foo")]), + // A bareword argument is a call of its own, not the callee. + ("foo(bar);", &[(BRACKETS, "foo(bar)"), (BAREWORD, "bar")]), + ("foo(1", &[(BAREWORD, "foo")]), + ]; + for (source, expected) in rows { + let (found, branches) = perl_calls(&format!("{source}\n")); + let expected = expected + .iter() + .map(|&(kind, text)| (kind.to_owned(), text.to_owned())) + .collect::>(); + assert_eq!(found, expected, "`{source}`"); + assert_eq!(branches, found.len(), "`{source}`: ABC disagrees"); + } +} + +/// An auto-quoted key is a string, so it reports the calls its quoted +/// twin does — none of its own. A word Perl does not quote keeps its +/// call: a package-qualified word, one with arguments, and a value. +/// Every count is checked against ABC's branches as well. +#[cfg(feature = "perl")] +#[test] +fn perl_bareword_key_is_no_call_1580() { + let rows = [ + ("my %h = (name => 1);", "my %h = ('name' => 1);", 0), + ("my %h = (-name => 1);", "my %h = ('-name' => 1);", 0), + ("my %h = (-size => 1);", "my %h = ('-size' => 1);", 0), + ("my $y = $h{-name};", "my $y = $h{'-name'};", 0), + ("my $y = $h{-size};", "my $y = $h{'-size'};", 0), + ("my %h = (1 - name => 2);", "my %h = (1 - 'name' => 2);", 0), + ("new(name => $x);", "new('name' => $x);", 1), + ("my %h = (Foo::Bar => 1);", "my %h = (Foo::Bar() => 1);", 1), + ("my %h = (name() => 1);", "my %h = (name(1) => 1);", 1), + ("my %h = (k => name);", "my %h = ('k' => name());", 1), + ]; + for (key, twin, calls) in rows { + let (found, branches) = perl_calls(&format!("{key}\n")); + let (twin_found, _) = perl_calls(&format!("{twin}\n")); + assert_eq!(found.len(), calls, "`{key}` reported {found:?}"); + assert_eq!(found.len(), twin_found.len(), "`{key}` against `{twin}`"); + assert_eq!(branches, calls, "`{key}`: ABC disagrees"); + } } From 07255db4ceb1aeaca2f21840beb5040188323e0c Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:12:46 -0700 Subject: [PATCH 18/34] fix(elixir): score receive like case A `receive` is a `case` over the mailbox, but modified cyclomatic and cognitive did not know the construct: a `receive` with any number of clauses sat at the base of both, and nothing inside it was nested. It now pays the container decision and +nesting as `case` does, in the block, keyword and bracketed spellings. Its `after` clause is the timeout, the path taken when no message clause matched in time. The compiler accepts exactly one, its head is a timeout rather than a pattern, and `try`'s `after` takes no clauses, so it is `receive`'s default arm and is free in every metric, as a `case`'s `_ ->` is. Standard cyclomatic and ABC counted it before. A `receive` holding only an `after` is a timed wait and scores nothing. Fixes #1581 --- big-code-analysis-book/src/metrics.md | 17 ++++- src/metrics/abc.rs | 6 +- src/metrics/cognitive/elixir.rs | 19 +++-- src/metrics/cyclomatic/elixir.rs | 21 +++-- src/metrics/elixir_decisions.rs | 85 ++++++++++++++++----- src/metrics/elixir_spelling_parity_tests.rs | 57 ++++++++++---- 6 files changed, 155 insertions(+), 50 deletions(-) diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index e99b6b9b..3bb3a92f 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -158,7 +158,7 @@ application would over-count. | Kotlin | A subject-less `when` arm scores through the predicate slot; a subject-ful arm scores per entry | A subject-less arm's condition (`when { x > 5 -> … }`) is an ordinary boolean expression, compiled as an `if` predicate, so the comparison inside it is already counted by the operator arms and a blanket per-entry increment double-counted it. A subject-ful arm (`when (x) { in 1..2 -> … }`) lists a pattern rather than an independent boolean expression, so the implicit `subject == pattern` is a decision nothing in the source spells and the entry itself pays for it (#1421). | | Ruby | A subject-less `case` scores each `when` pattern as an `\|\|` operand; a subject-ful `case` scores per clause | A subject-less clause (`case; when x > 5`) lists ordinary boolean expressions, evaluated like an `if` predicate, and several patterns (`when a, b`) are an implicit `\|\|`. Each pattern therefore scores once unless its own operator already counted it, so `when x > 5` scores 1 like `if x > 5`, and `when a, b` scores 2 like `if a \|\| b`. Before this, a blanket per-clause increment counted a comparison twice. A subject-ful clause (`case x; when 1`) matches each pattern with an implicit `pattern === x` that nothing in the source spells, so the clause pays once (#1453). | | Go | A tagless `switch` scores each case expression as an `\|\|` operand; a tagged `switch` scores per case | A tagless switch (`switch { case x > 5: }`, with or without an init statement) is `switch true`: each case expression is an ordinary boolean scored by the same slot an `if` predicate uses, and a list (`case a, b:`) is an implicit `\|\|`. So `case x > 5:` scores 1 like `if x > 5`, and `case a, b:` scores 2 like `if a \|\| b`. Before this, a blanket per-case increment counted a comparison twice. A tagged case (`switch x { case 1: }`) compares with an implicit `x == 1` that nothing in the source spells, so the case pays once; type-switch and `select` cases are unchanged (#1523). | -| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive` (its `after` timeout included), of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). A bracketed keyword list in last position (`if(c, [do: 1, else: 2])`) is the same argument as the bare one and scores the same, as does a quoted key (`"else": 2`); a list anywhere else, or one with a non-pair item, holds no sections. A `Kernel.&&` / `Kernel.\|\|` remote call scores its operands as `a && b` / `a \|\| b` does, the piped value of `a \|> Kernel.\|\|(b)` included (#1574). | +| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive`, of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). A bracketed keyword list in last position (`if(c, [do: 1, else: 2])`) is the same argument as the bare one and scores the same, as does a quoted key (`"else": 2`); a list anywhere else, or one with a non-pair item, holds no sections. A `Kernel.&&` / `Kernel.\|\|` remote call scores its operands as `a && b` / `a \|\| b` does, the piped value of `a \|> Kernel.\|\|(b)` included (#1574). A `receive`'s `after` clause is its timeout, the path taken when no message clause matched in time, so it is the construct's default arm and scores nothing, as `_ ->` does in a `case` (#1581). | | C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these brackets as an operator (a type-enclosing pair as one `<>` since #1559 and #1567), and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | | C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | @@ -367,7 +367,8 @@ Sonar ecosystem. ### Per-language deviations - **Elixir** does not score recursion or jump statements. Elixir - control flow (`if` / `unless` / `cond` / `case` / `with`) is built + control flow (`if` / `unless` / `cond` / `case` / `with` / + `receive`) is built from macro-shaped `Call` nodes rather than dedicated grammar productions, and the language has no `break` / `continue` / `goto`; the implementation therefore scores only the @@ -380,7 +381,10 @@ Sonar ecosystem. is. An `if` / `unless` / `try` `else` still adds +1. The keyword forms (`else:`, `rescue:`, `catch:`, bare or bracketed) score as their `do … end` blocks do, and `Kernel.||(a, b)` is a boolean - sequence like `a || b` (#1574). + sequence like `a || b` (#1574). A `receive` is a `case` over the + mailbox and pays +nesting as one; its `after` timeout adds nothing, + and a `receive` holding only an `after` is a timed wait that scores + nothing (#1581). - For every language with a syntactic function-definition node, a nested function — a local function, or a method on a local / inner class — **resets the nesting counter to zero** at its @@ -524,6 +528,13 @@ one of them is a decision: `else _ -> 0` selects nothing, like a plain or diverges, so it scores one in both variants, as its `if let` twin does. Before #1542 it scored nothing. +Elixir's `receive` is a `case` over the mailbox, and modified scores +its clauses once, as it does a `case`'s. Its `after` clause is the +timeout: the path taken when no clause matched in time. That makes it +the construct's default arm, so it scores nothing in either variant, as +`_ ->` does in a `case`. A `receive` holding only an `after` is a timed +wait and scores nothing at all (#1581). + ### Counting Rust's `?` operator By default Rust's `?` operator (the `try_expression` grammar node) diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index fa690ff6..e42ef0f9 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -11716,7 +11716,9 @@ end // clause's flat one beside its `>`, `rescue_in` a sole pattern that // scores itself (the `in` arm), `rescue_alias` a sole pattern the // clause slot pays for, `catch_two` a multi-pattern clause, `recv` a - // `receive`'s `after` timeout clause, and `with_else` the `else` of + // `receive`'s `after` timeout clause — free since #1581, as the + // default arm, so `recv` scores what `case2` does — and `with_else` + // the `else` of // a `with`, whose clauses pay rather than a Rule 5 `else`. Its `<-` // clause pays one more since #1535, in both metrics. #[cfg(feature = "elixir")] @@ -11791,7 +11793,7 @@ end ("case_default", 2, 3), ("case_guard", 2, 3), ("with_else", 2, 3), - ("recv", 3, 4), + ("recv", 2, 3), ("rescue_in", 1, 2), ("rescue_alias", 1, 2), ("catch_two", 1, 2), diff --git a/src/metrics/cognitive/elixir.rs b/src/metrics/cognitive/elixir.rs index 9a395f37..26761f83 100644 --- a/src/metrics/cognitive/elixir.rs +++ b/src/metrics/cognitive/elixir.rs @@ -13,7 +13,7 @@ use super::*; use crate::lang_helpers::elixir::elixir_applying_remote_call; -use crate::metrics::elixir_decisions::elixir_keyword_pair_section; +use crate::metrics::elixir_decisions::{elixir_keyword_pair_section, elixir_receive_dispatches}; /// Whether a Call spelling `keyword` takes `try`'s sections — `else`, /// `rescue`, `catch` — in its keyword form: `try` itself, and the @@ -85,7 +85,7 @@ fn compute_elixir_remote_booleans<'a>( impl Cognitive for ElixirCode { // Elixir control flow is macro-shaped: `if`, `unless`, `case`, - // `cond`, `with`, `for`, `while`, and `try` each surface as a + // `cond`, `with`, `receive`, `for`, `while`, and `try` each surface as a // `Call` node whose `target` Identifier text spells the keyword. // We classify the Call once on entry (raising nesting), then let // the structural `Else` token and `Rescue` / `Catch` blocks inside @@ -96,9 +96,10 @@ impl Cognitive for ElixirCode { // - `if` / `unless` / `for` / `while`: single-branch control flow, // `+nesting`. Their `else` (token `Elixir::Else` inside an // `ElseBlock`) adds `+1` without nesting, matching Java. - // - `case` / `cond` / `with`: switch-/multi-arm, `+nesting` once on - // the container. Individual `stab_clause` arms do NOT add extra - // cost (matches Java `SwitchBlock` / `case:` rule). + // - `case` / `cond` / `with` / `receive`: switch-/multi-arm, + // `+nesting` once on the container. Individual `stab_clause` arms + // do NOT add extra cost (matches Java `SwitchBlock` / `case:` + // rule). // - `try`: `+0` on the container itself — it is a wrapper, and its // `rescue` / `catch` arms surface as `RescueBlock` / `CatchBlock` // and each one adds `+nesting`, matching Java's `CatchClause` @@ -138,6 +139,14 @@ impl Cognitive for ElixirCode { Some("if" | "unless" | "for" | "while" | "case" | "cond" | "with") => { increase_nesting(stats, &mut nesting); } + // A `case` over the mailbox, paying as one (#1581) when + // it has message clauses; one holding only an `after` + // timeout is a timed wait, the idiomatic sleep, and + // branches on nothing. The `after` adds nothing either + // way: it is the default arm, which `case` does not pay. + Some("receive") if elixir_receive_dispatches(node, code) => { + increase_nesting(stats, &mut nesting); + } // `try` is intentionally absent: it is a wrapper for // `rescue` / `catch` arms (each of which earns its own // +nesting via `RescueBlock` / `CatchBlock`). Adding the diff --git a/src/metrics/cyclomatic/elixir.rs b/src/metrics/cyclomatic/elixir.rs index 4cc0142b..b6f0b46e 100644 --- a/src/metrics/cyclomatic/elixir.rs +++ b/src/metrics/cyclomatic/elixir.rs @@ -9,8 +9,8 @@ use super::*; use crate::lang_helpers::elixir::{elixir_applying_operator, elixir_applying_remote_call}; use crate::metrics::elixir_decisions::{ - elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_when_is_guard, - elixir_with_else_dispatches, + elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_receive_dispatches, + elixir_when_is_guard, elixir_with_else_dispatches, }; impl Cyclomatic for ElixirCode { @@ -24,7 +24,7 @@ impl Cyclomatic for ElixirCode { // The split between standard and modified CCN mirrors the C-family // case/switch treatment: per-arm `stab_clause` nodes contribute // standard, while the multi-arm container Calls (`case`/`cond`/ - // `try`) contribute modified. Single-branch keyword Calls + // `try`/`receive`) contribute modified. Single-branch keyword Calls // (`if`/`unless`/`for`/`while`) contribute to both, as do a `with`'s // `<-` clauses and short-circuit booleans (`&&`, `||`, `and`, `or`). fn compute<'a>( @@ -38,9 +38,9 @@ impl Cyclomatic for ElixirCode { match node.kind_id().into() { // Per-arm decisions: each `stab_clause` is one arm of a - // `case`/`cond`/`with`/anonymous-fn body or a `rescue`/ - // `catch` handler. Standard-only: modified counts a - // `case`/`cond`/`try` container Call once, a `with` once + // `case`/`cond`/`with`/`receive`/anonymous-fn body or a + // `rescue`/`catch` handler. Standard-only: modified counts a + // `case`/`cond`/`try`/`receive` container Call once, a `with` once // only when its `else` dispatches (the `"with"` arm below), // and gives a multi-clause `fn` or `reduce` nothing. // @@ -152,6 +152,15 @@ impl Cyclomatic for ElixirCode { "case" | "cond" | "try" => { stats.cyclomatic_modified += 1.; } + // A `receive` is a `case` over the mailbox and + // pays as one (#1581), when it has message + // clauses to pick between: one holding only an + // `after` timeout is a timed wait. The timeout + // clause is the default arm and free in standard + // (`elixir_clause_is_decision`). + "receive" if elixir_receive_dispatches(node, code) => { + stats.cyclomatic_modified += 1.; + } // A `with`'s decisions are its `<-` clauses, // counted above in both tiers. What modified // collapses is its `else`, the dispatch over diff --git a/src/metrics/elixir_decisions.rs b/src/metrics/elixir_decisions.rs index d1a425bd..75e1213c 100644 --- a/src/metrics/elixir_decisions.rs +++ b/src/metrics/elixir_decisions.rs @@ -333,6 +333,15 @@ fn elixir_is_first_clause<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> /// `fn`'s first, except the construct's free default clause /// ([`elixir_is_default_clause`]). /// +/// A `receive`'s `after` clause is free too, in every spelling: it is +/// the timeout, the path taken when no message clause matched in time, +/// which makes it `receive`'s default arm — the role a bare `_ ->` plays +/// in a `case` (#1581). It is never a dispatch of its own: the compiler +/// accepts exactly one `after` clause, and its head is a timeout value, +/// not a pattern that can fail. No other construct puts a clause there, +/// since `try`'s `after` takes a plain body, so the section is simply +/// not among those whose clauses count. +/// /// An anonymous fn's head clause is the closure's definition, not a /// dispatch: the closure opens its own function space whose base path /// already covers it, so counting it too would score a trivial @@ -365,15 +374,10 @@ pub(super) fn elixir_clause_is_decision<'a>( let in_section = up.next().is_some_and(|parent| { matches!( parent.kind_id().into(), - E::DoBlock - | E::ElseBlock - | E::AfterBlock - | E::RescueBlock - | E::CatchBlock - | E::AnonymousFunction + E::DoBlock | E::ElseBlock | E::RescueBlock | E::CatchBlock | E::AnonymousFunction ) || (parent.kind_id() == E::Block as u16 && up.next().is_some_and(|pair| { - elixir_pair_key(&pair, code).is_some_and(|key| SECTION_KEYS.contains(&key)) + elixir_pair_key(&pair, code).is_some_and(|key| DECISION_SECTION_KEYS.contains(&key)) })) }); if !in_section { @@ -405,30 +409,71 @@ pub(super) fn elixir_clause_is_decision<'a>( /// under a `fn` or `reduce:`, so [`elixir_clause_is_decision`] agrees /// clause by clause with the test below. pub(super) fn elixir_with_else_dispatches(call: &Node, code: &[u8]) -> bool { + elixir_section_has_clause(call, code, Section::Else, |clause| { + !elixir_sole_unguarded_pattern(clause) + .is_some_and(|pattern| elixir_is_bare_wildcard(&pattern, code)) + }) +} + +/// Whether a `receive` Call dispatches over the mailbox: whether its +/// `do` section holds a message clause, in any spelling. One holding +/// only an `after` timeout (`receive do after 100 -> :ok end`, the +/// idiomatic sleep) is a timed wait that picks between nothing, so it +/// pays neither modified cyclomatic's container decision nor cognitive's +/// +nesting — as a `with` whose `else` picks nothing pays no modified +/// decision ([`elixir_with_else_dispatches`]). Any clause counts, a bare +/// `_ ->` included, because `case x do _ -> 0 end` pays its container +/// too (#1581). +pub(super) fn elixir_receive_dispatches(call: &Node, code: &[u8]) -> bool { + elixir_section_has_clause(call, code, Section::Do, |_| true) +} + +/// A clause section [`elixir_section_has_clause`] reads. +#[derive(Clone, Copy)] +enum Section { + Do, + Else, +} + +/// Whether `call`'s `section` holds a `stab_clause` that `accept`s: the +/// block form's `do_block`, or the `else_block` inside it, or the +/// keyword form's `do:` / `else:` value, bare or bracketed +/// ([`elixir_trailing_keywords`]). The block form's `else_block` is the +/// `do_block`'s child, never the Call's. +fn elixir_section_has_clause( + call: &Node, + code: &[u8], + section: Section, + accept: impl Fn(&Node) -> bool, +) -> bool { use Elixir as E; + let key = match section { + Section::Do => "do", + Section::Else => "else", + }; call.children().any(|child| { - let section = match child.kind_id().into() { - E::DoBlock => child + let holder = match (child.kind_id().into(), section) { + (E::DoBlock, Section::Do) => Some(child), + (E::DoBlock, Section::Else) => child .children() .find(|c| c.kind_id() == E::ElseBlock as u16), - _ if child.kind() == ARGUMENTS => elixir_keyword_section(&child, code, "else"), + _ if child.kind() == ARGUMENTS => elixir_keyword_section(&child, code, key), _ => None, }; - section.is_some_and(|section| { - section.children().any(|clause| { - clause.kind_id() == E::StabClause as u16 - && !elixir_sole_unguarded_pattern(&clause) - .is_some_and(|pattern| elixir_is_bare_wildcard(&pattern, code)) - }) + holder.is_some_and(|holder| { + holder + .children() + .any(|clause| clause.kind_id() == E::StabClause as u16 && accept(&clause)) }) }) } -/// The keyword spellings of the clause sections a `do … end` block -/// spells as `do_block` / `else_block` / `after_block` / `rescue_block` -/// / `catch_block`. -const SECTION_KEYS: &[&str] = &["do", "else", "after", "rescue", "catch"]; +/// The keyword spellings of the clause sections whose clauses are +/// decisions, which a `do … end` block spells as `do_block` / +/// `else_block` / `rescue_block` / `catch_block`. Not `after`: see +/// [`elixir_clause_is_decision`]. +const DECISION_SECTION_KEYS: &[&str] = &["do", "else", "rescue", "catch"]; /// The key a keyword-list `pair` spells, without its colon: `do` for /// both `do: x` and the quoted `"do": x`, which Elixir reads as the same diff --git a/src/metrics/elixir_spelling_parity_tests.rs b/src/metrics/elixir_spelling_parity_tests.rs index 9fff50e0..de4cbefe 100644 --- a/src/metrics/elixir_spelling_parity_tests.rs +++ b/src/metrics/elixir_spelling_parity_tests.rs @@ -325,20 +325,49 @@ fn elixir_for_reduce_head_is_free_in_every_spelling() { ); } -/// `receive` agrees across spellings, but its modified and cognitive -/// scores are wrong in all of them, so the group asserts only agreement. -/// That agreement predates #1574 — its clauses score through the -/// owner-blind section gate (#1583), not the trailing-list lookup — so -/// this guards the spellings drifting apart, not that fix. -// FIXME(#1581): pin the values once `receive` pays like `case`. +/// A `receive` is a `case` over the mailbox and scores as one in every +/// spelling (#1581): before, modified cyclomatic and cognitive did not +/// know the construct, so it sat at the base of both, and its `after` +/// timeout counted as one more clause. The timeout is the path taken +/// when no clause matched in time — `receive`'s default arm, which +/// `case`'s `_ ->` twin shows is free — so with it the `receive` still +/// scores what the two-clause `case` does. #[cfg(feature = "elixir")] #[test] -fn elixir_receive_scores_alike_in_every_spelling() { - let block = measure("receive do {:a, x} -> x; :b -> 0 after 10 -> :t end"); - for spelling in [ - "receive(do: ({:a, x} -> x; :b -> 0), after: (10 -> :t))", - "receive([do: ({:a, x} -> x; :b -> 0), after: (10 -> :t)])", - ] { - assert_eq!(measure(spelling), block, "`{spelling}`"); - } +fn elixir_receive_scores_like_case() { + assert_group( + "case x do {:a, y} -> y; :b -> 0 end", + [3, 2, 2, 1], + &[ + "receive do {:a, y} -> y; :b -> 0 end", + "receive do {:a, y} -> y; :b -> 0 after 10 -> :t end", + "receive(do: ({:a, y} -> y; :b -> 0), after: (10 -> :t))", + "receive([do: ({:a, y} -> y; :b -> 0), after: (10 -> :t)])", + ], + ); + // The container raises nesting for what its clauses hold. + assert_group( + "case x do {:a, y} -> (if y do 1 end) end", + [3, 3, 2, 3], + &[ + "receive do {:a, y} -> (if y do 1 end) after 10 -> :t end", + "receive(do: ({:a, y} -> if(y, do: 1)), after: (10 -> :t))", + ], + ); + // A lone catch-all is free in standard and still pays the container. + assert_group( + "case x do _ -> 0 end", + [1, 2, 0, 1], + &["receive do _ -> 0 end", "receive(do: (_ -> 0))"], + ); + // With no message clause it is a timed wait, the sleep it spells. + assert_group( + "Process.sleep(10)", + [1, 1, 0, 0], + &[ + "receive do after 10 -> :t end", + "receive(after: (10 -> :t))", + "receive([after: (10 -> :t)])", + ], + ); } From 202d5d5ab6fb260edf70d383bb87f03318249904 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:16:34 -0700 Subject: [PATCH 19/34] fix(elixir): score Kernel.and and Kernel.or `Kernel.and(a, b)` and `Kernel.or(a, b)` expand the same short-circuit macros as `a and b` / `a or b`, but scored no decision in any metric. #1574 routed `Kernel.&&` / `Kernel.||` through the remote-call predicate; the word forms never reached it, because the grammar names them with a plain `identifier` rather than an operator token. The `Kernel` lookup now returns that identifier when its bytes spell `and` or `or`, and one helper keys every spelling of a short-circuit operation to its symbol, so cognitive treats `a or Kernel.||(b, c)` as one sequence. Cyclomatic, ABC and cognitive each read the identifier beside the token. `Kernel.not` and `Kernel.!` stay non-decisions, and a user module's `Foo.and` stays an ordinary call. Fixes #1582 --- .../src/lang_helpers/elixir.rs | 75 ++++++++++++++++--- big-code-analysis-book/src/metrics.md | 9 ++- src/metrics/abc/elixir.rs | 42 +++++++---- src/metrics/cognitive/elixir.rs | 33 ++++---- src/metrics/cyclomatic/elixir.rs | 9 +++ src/metrics/elixir_spelling_parity_tests.rs | 59 +++++++++++++++ 6 files changed, 185 insertions(+), 42 deletions(-) diff --git a/big-code-analysis-ast/src/lang_helpers/elixir.rs b/big-code-analysis-ast/src/lang_helpers/elixir.rs index f09ea577..307f546f 100644 --- a/big-code-analysis-ast/src/lang_helpers/elixir.rs +++ b/big-code-analysis-ast/src/lang_helpers/elixir.rs @@ -127,9 +127,13 @@ pub fn elixir_applying_operator<'a>( /// may define `||/2` as an ordinary function, and `Foo.||(a, b)` neither /// short-circuits nor decides. /// -/// Only an operator the grammar lexes as a token inside the call's -/// `operator_identifier` is found. `Kernel.and(a, b)` spells its name as -/// a plain `identifier`, so it is not; nor is the childless `..`. +/// The grammar lexes a symbolic name as a token inside the call's +/// `operator_identifier`, which is what is returned. `Kernel.and(a, b)` +/// and `Kernel.or(a, b)` spell their names as a plain `identifier`, with +/// no `And` / `Or` token to find, so that identifier is returned instead +/// (#1582); every other word name (`Kernel.not`, `Kernel.in`) is a +/// function or decides nothing, and is not found. Nor is the childless +/// `..`. /// /// Matched by rule name: `call`, `dot` and `arguments` are aliased to /// several ids each (grammar-dispatch §1). @@ -147,23 +151,72 @@ pub fn elixir_kernel_operator_token<'a>(call: &Node<'a>, code: &[u8]) -> Option< ) }) })?; - dot.child_by_field_name("right") - .filter(|name| name.kind() == "operator_identifier")? - .child(0) + let name = dot.child_by_field_name("right")?; + if name.kind() == "operator_identifier" { + return name.child(0); + } + (name.kind_id() == Elixir::Identifier as u16 + && matches!( + code.get(name.start_byte()..name.end_byte()), + Some(b"and" | b"or") + )) + .then_some(name) } -/// The `Kernel` remote call that applies `op`, an operator token — the +/// The short-circuit operation an operator token kind applies, keyed by +/// its symbol: `&&` for `&&` and `and`, `||` for `||` and `or`. A word +/// form demands a boolean left operand, but it is the same conjunction or +/// disjunction to read, so one sequence key serves both spellings. +/// `None` for any other kind. +#[must_use] +pub fn elixir_short_circuit_kind(kind_id: u16) -> Option { + match kind_id.into() { + Elixir::AMPAMP | Elixir::And => Some(Elixir::AMPAMP), + Elixir::PIPEPIPE | Elixir::Or => Some(Elixir::PIPEPIPE), + _ => None, + } +} + +/// [`elixir_short_circuit_kind`] for an operator's name as +/// [`elixir_kernel_operator_token`] returns it: a token, or the +/// `identifier` spelling `Kernel.and` / `Kernel.or` (#1582), whose +/// bytes decide. Ask it only of such a name — an identifier is an `and` +/// operator nowhere else. +#[must_use] +pub fn elixir_short_circuit_name(name: &Node, code: &[u8]) -> Option { + if name.kind_id() != Elixir::Identifier as u16 { + return elixir_short_circuit_kind(name.kind_id()); + } + match code.get(name.start_byte()..name.end_byte())? { + b"and" => Some(Elixir::AMPAMP), + b"or" => Some(Elixir::PIPEPIPE), + _ => None, + } +} + +/// The `Kernel` remote call that applies `op`, an operator's name — the /// call [`elixir_kernel_operator_token`] answers `op` for — or `None` -/// when `op` sits anywhere else. The token's parents are the -/// `operator_identifier`, then the `dot`, then the call. +/// when `op` sits anywhere else. A token's parents are the +/// `operator_identifier`, then the `dot`, then the call; the +/// `identifier` naming `Kernel.and` sits on the `dot` directly. Any +/// other node fails on the first one or two parents' kinds, so the +/// check is cheap enough to ask of every identifier a walk meets. #[must_use] pub fn elixir_applying_remote_call<'a>( op: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>, ) -> Option> { - let (call, _) = ancestors.iter(op).nth(2)?; + let mut up = ancestors.iter(op).map(|(ancestor, _)| ancestor); + let mut dot = up.next()?; + if dot.kind() == "operator_identifier" { + dot = up.next()?; + } + if dot.kind() != "dot" { + return None; + } + let call = up.next()?; elixir_kernel_operator_token(&call, code) - .is_some_and(|token| token.id() == op.id()) + .is_some_and(|name| name.id() == op.id()) .then_some(call) } diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 3bb3a92f..28374a12 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -158,7 +158,7 @@ application would over-count. | Kotlin | A subject-less `when` arm scores through the predicate slot; a subject-ful arm scores per entry | A subject-less arm's condition (`when { x > 5 -> … }`) is an ordinary boolean expression, compiled as an `if` predicate, so the comparison inside it is already counted by the operator arms and a blanket per-entry increment double-counted it. A subject-ful arm (`when (x) { in 1..2 -> … }`) lists a pattern rather than an independent boolean expression, so the implicit `subject == pattern` is a decision nothing in the source spells and the entry itself pays for it (#1421). | | Ruby | A subject-less `case` scores each `when` pattern as an `\|\|` operand; a subject-ful `case` scores per clause | A subject-less clause (`case; when x > 5`) lists ordinary boolean expressions, evaluated like an `if` predicate, and several patterns (`when a, b`) are an implicit `\|\|`. Each pattern therefore scores once unless its own operator already counted it, so `when x > 5` scores 1 like `if x > 5`, and `when a, b` scores 2 like `if a \|\| b`. Before this, a blanket per-clause increment counted a comparison twice. A subject-ful clause (`case x; when 1`) matches each pattern with an implicit `pattern === x` that nothing in the source spells, so the clause pays once (#1453). | | Go | A tagless `switch` scores each case expression as an `\|\|` operand; a tagged `switch` scores per case | A tagless switch (`switch { case x > 5: }`, with or without an init statement) is `switch true`: each case expression is an ordinary boolean scored by the same slot an `if` predicate uses, and a list (`case a, b:`) is an implicit `\|\|`. So `case x > 5:` scores 1 like `if x > 5`, and `case a, b:` scores 2 like `if a \|\| b`. Before this, a blanket per-case increment counted a comparison twice. A tagged case (`switch x { case 1: }`) compares with an implicit `x == 1` that nothing in the source spells, so the case pays once; type-switch and `select` cases are unchanged (#1523). | -| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive`, of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). A bracketed keyword list in last position (`if(c, [do: 1, else: 2])`) is the same argument as the bare one and scores the same, as does a quoted key (`"else": 2`); a list anywhere else, or one with a non-pair item, holds no sections. A `Kernel.&&` / `Kernel.\|\|` remote call scores its operands as `a && b` / `a \|\| b` does, the piped value of `a \|> Kernel.\|\|(b)` included (#1574). A `receive`'s `after` clause is its timeout, the path taken when no message clause matched in time, so it is the construct's default arm and scores nothing, as `_ ->` does in a `case` (#1581). | +| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive`, of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). A bracketed keyword list in last position (`if(c, [do: 1, else: 2])`) is the same argument as the bare one and scores the same, as does a quoted key (`"else": 2`); a list anywhere else, or one with a non-pair item, holds no sections. A `Kernel.&&` / `Kernel.\|\|` remote call scores its operands as `a && b` / `a \|\| b` does, the piped value of `a \|> Kernel.\|\|(b)` included (#1574), and so do `Kernel.and` / `Kernel.or` (#1582). A `receive`'s `after` clause is its timeout, the path taken when no message clause matched in time, so it is the construct's default arm and scores nothing, as `_ ->` does in a `case` (#1581). | | C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these brackets as an operator (a type-enclosing pair as one `<>` since #1559 and #1567), and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | | C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | @@ -381,7 +381,8 @@ Sonar ecosystem. is. An `if` / `unless` / `try` `else` still adds +1. The keyword forms (`else:`, `rescue:`, `catch:`, bare or bracketed) score as their `do … end` blocks do, and `Kernel.||(a, b)` is a boolean - sequence like `a || b` (#1574). A `receive` is a `case` over the + sequence like `a || b` (#1574), as `Kernel.or(a, b)` is like + `a or b` (#1582). A `receive` is a `case` over the mailbox and pays +nesting as one; its `after` timeout adds nothing, and a `receive` holding only an `after` is a timed wait that scores nothing (#1581). @@ -917,7 +918,9 @@ tokens involved the same way they spell real operators: what `&foo/2` does. A captured `&&` / `||` / `and` / `or` is no decision either (#1534), but the remote call applies the operator, so `Kernel.||(a, b)` scores what `a || b` does in cyclomatic, ABC and - cognitive alike, and a user module's `Foo.||(a, b)` in none (#1574). This + cognitive alike, and a user module's `Foo.||(a, b)` in none (#1574). + `Kernel.and(a, b)` and `Kernel.or(a, b)` decide the same way, though + their names are plain identifiers with no operator token (#1582). This differs from a C# `operator <`, whose `<` has no wrapper and so stays the operator the rule above makes it. diff --git a/src/metrics/abc/elixir.rs b/src/metrics/abc/elixir.rs index c0749991..c9d4b74a 100644 --- a/src/metrics/abc/elixir.rs +++ b/src/metrics/abc/elixir.rs @@ -15,7 +15,7 @@ use super::{ }; use crate::lang_helpers::elixir::{ elixir_applying_operator, elixir_applying_remote_call, elixir_call_keyword, - elixir_kernel_operator_token, + elixir_kernel_operator_token, elixir_short_circuit_name, }; use crate::metrics::elixir_decisions::{ elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_keyword_section, @@ -78,8 +78,9 @@ fn elixir_binary_operator_is(node: &Node, accept: impl Fn(Elixir) -> bool) -> bo // each operand through `elixir_count_condition` — so every operand // either pays there or is one of these, and a chain always carries at // least one condition of its own. A short-circuit `Kernel` remote call -// (`Kernel.||(a, b)`) is that chain spelled as a call, and its operands -// pay through `elixir_count_remote_chain` in the same way (#1574). +// (`Kernel.||(a, b)`, `Kernel.or(a, b)`) is that chain spelled as a +// call, and its operands pay through `elixir_count_remote_chain` in the +// same way (#1574, #1582). // // The operator list must agree with the token arms in `compute`: an // operator listed here and counted nowhere would leave its slot at zero, @@ -88,7 +89,8 @@ fn elixir_condition_scores_itself(expr: &Node, code: &[u8]) -> bool { use Elixir as E; elixir_kernel_operator_token(expr, code) - .is_some_and(|op| matches!(op.kind_id().into(), E::AMPAMP | E::PIPEPIPE)) + .and_then(|name| elixir_short_circuit_name(&name, code)) + .is_some() || elixir_binary_operator_is(expr, |op| { matches!( op, @@ -204,12 +206,23 @@ fn elixir_has_else(call: &Node, code: &[u8]) -> bool { // Which side of the pipe the call sits on is not checked: in // `Kernel.||(a, b) |> f()` the piped value is the call itself, which // `elixir_condition_scores_itself` already says pays nothing. +// +// `op` is the operator's name, a token or the `identifier` of +// `Kernel.and` / `Kernel.or` (#1582); any other node scores nothing. fn elixir_count_remote_chain<'a>( - call: &Node<'a>, - code: &[u8], - mut up: impl Iterator>, + op: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, conditions: &mut f64, ) { + let Some(call) = elixir_applying_remote_call(op, code, ancestors) else { + return; + }; + let mut up = ancestors + .iter(op) + .map(|(ancestor, _)| ancestor) + .skip_while(|ancestor| ancestor.id() != call.id()) + .skip(1); let score = |operand: &Node, conditions: &mut f64| { elixir_count_condition(operand, code, conditions); }; @@ -490,15 +503,16 @@ impl Abc for ElixirCode { |operand, conditions| elixir_count_condition(operand, code, conditions), &mut stats.conditions, ); - } else if let Some(call) = elixir_applying_remote_call(node, code, ancestors) { - let above_call = ancestors - .iter(node) - .map(|(ancestor, _)| ancestor) - .skip_while(|ancestor| ancestor.id() != call.id()) - .skip(1); - elixir_count_remote_chain(&call, code, above_call, &mut stats.conditions); + } else { + elixir_count_remote_chain(node, code, ancestors, &mut stats.conditions); } } + // `Kernel.and(a, b)` / `Kernel.or(a, b)` name the operator + // with an `identifier`, so the token arm above never sees + // them; they are the same chain spelled as a call (#1582). + E::Identifier => { + elixir_count_remote_chain(node, code, ancestors, &mut stats.conditions); + } E::StabClause if elixir_clause_is_decision(node, code, ancestors) => { elixir_count_clause(node, code, &mut stats.conditions); } diff --git a/src/metrics/cognitive/elixir.rs b/src/metrics/cognitive/elixir.rs index 26761f83..6a0e7bc0 100644 --- a/src/metrics/cognitive/elixir.rs +++ b/src/metrics/cognitive/elixir.rs @@ -12,7 +12,9 @@ )] use super::*; -use crate::lang_helpers::elixir::elixir_applying_remote_call; +use crate::lang_helpers::elixir::{ + elixir_applying_remote_call, elixir_short_circuit_kind, elixir_short_circuit_name, +}; use crate::metrics::elixir_decisions::{elixir_keyword_pair_section, elixir_receive_dispatches}; /// Whether a Call spelling `keyword` takes `try`'s sections — `else`, @@ -55,31 +57,31 @@ fn elixir_section<'a>( /// Folds an Elixir `BinaryOperator`'s short-circuit operator children /// into the boolean-sequence counter — Elixir has four (`&&`, `||`, /// `and`, `or`). Each word form is keyed to its symbol (see -/// [`compute_booleans_keyed`]): `and` / `or` demand a boolean left -/// operand, but they are the same conjunction / disjunction to read. +/// [`compute_booleans_keyed`] and [`elixir_short_circuit_kind`]). fn compute_elixir_booleans(node: &Node, stats: &mut Stats) { - compute_booleans_keyed(node, stats, |id| match id.into() { - Elixir::AMPAMP | Elixir::And => Some(Elixir::AMPAMP as u16), - Elixir::PIPEPIPE | Elixir::Or => Some(Elixir::PIPEPIPE as u16), - _ => None, + compute_booleans_keyed(node, stats, |id| { + elixir_short_circuit_kind(id).map(|key| key as u16) }); } /// [`compute_elixir_booleans`] for a short-circuit `Kernel` remote call -/// (`Kernel.||(a, b)`), whose operator token sits in the call's name -/// rather than among a `binary_operator`'s children. The sequence is -/// keyed on the call's span, as a `binary_operator`'s is on its own. +/// (`Kernel.||(a, b)`, `Kernel.or(a, b)`), whose operator sits in the +/// call's name rather than among a `binary_operator`'s children. The +/// sequence is keyed on the call's span, as a `binary_operator`'s is on +/// its own, and on the same symbol, so `a or Kernel.||(b, c)` is one run. fn compute_elixir_remote_booleans<'a>( op: &Node<'a>, code: &'a [u8], ancestors: Ancestors<'a, '_>, stats: &mut Stats, ) { - if let Some(call) = elixir_applying_remote_call(op, code, ancestors) { + if let Some(call) = elixir_applying_remote_call(op, code, ancestors) + && let Some(key) = elixir_short_circuit_name(op, code) + { stats.structural = stats .boolean_seq - .eval_based_on_prev(op.kind_id(), call.end_byte(), stats.structural); + .eval_based_on_prev(key as u16, call.end_byte(), stats.structural); } } @@ -110,7 +112,8 @@ impl Cognitive for ElixirCode { // does NOT bump — see the arm comment below for why. // - `AnonymousFunction` (`fn x -> y end`): lambda nesting bumps. // - `&&` / `||` / `and` / `or`: boolean sequence cost, as is a - // `Kernel.&&` / `Kernel.||` remote call. + // `Kernel.&&` / `Kernel.||` / `Kernel.and` / `Kernel.or` remote + // call. // // Limitations: // - `Enum.reduce` / `Enum.map` and friends are higher-order function @@ -242,7 +245,9 @@ impl Cognitive for ElixirCode { // `binary_operator`'s operator is keyed on the operator's. // `Foo.||(a, b)` is an ordinary call and is not matched; the // token inside a `binary_operator` is the arm above's. - E::AMPAMP | E::PIPEPIPE => { + // `Kernel.and` / `Kernel.or` name the operator with an + // `identifier` instead of a token (#1582). + E::AMPAMP | E::PIPEPIPE | E::Identifier => { compute_elixir_remote_booleans(node, code, ancestors, stats); } _ => {} diff --git a/src/metrics/cyclomatic/elixir.rs b/src/metrics/cyclomatic/elixir.rs index b6f0b46e..7a3af483 100644 --- a/src/metrics/cyclomatic/elixir.rs +++ b/src/metrics/cyclomatic/elixir.rs @@ -131,6 +131,15 @@ impl Cyclomatic for ElixirCode { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } + // `Kernel.and(a, b)` / `Kernel.or(a, b)` name the operator + // with an `identifier` rather than a token, so the arm above + // never sees them; they expand the same short-circuit macro + // and decide as `a and b` does (#1582). Only those two names + // are found, so no other identifier passes the guard. + E::Identifier if elixir_applying_remote_call(node, code, ancestors).is_some() => { + stats.cyclomatic += 1.; + stats.cyclomatic_modified += 1.; + } E::Call => { if let Some(target) = node.child_by_field_name("target") && target.kind_id() == E::Identifier diff --git a/src/metrics/elixir_spelling_parity_tests.rs b/src/metrics/elixir_spelling_parity_tests.rs index de4cbefe..6bb5aa26 100644 --- a/src/metrics/elixir_spelling_parity_tests.rs +++ b/src/metrics/elixir_spelling_parity_tests.rs @@ -106,6 +106,65 @@ fn elixir_kernel_remote_call_scores_like_the_operator() { ); } +/// `Kernel.and` / `Kernel.or` expand the same short-circuit macros as +/// `a and b` / `a or b`, but the grammar names them with a plain +/// `identifier` rather than an operator token, so before #1582 they +/// scored nothing in any metric. Each word form keys the cognitive +/// sequence its symbol does, so a mixed run of `or` and `||` is one +/// sequence and a change of operation opens a second. +#[cfg(feature = "elixir")] +#[test] +fn elixir_kernel_word_operator_scores_like_the_operator() { + assert_group( + "a and b", + [2, 2, 2, 1], + &[ + "Kernel.and(a, b)", + "Elixir.Kernel.and(a, b)", + "a |> Kernel.and(b)", + ], + ); + assert_group("a or b", [2, 2, 2, 1], &["Kernel.or(a, b)"]); + assert_group( + "a or (b or c)", + [3, 3, 3, 1], + &["Kernel.or(a, Kernel.or(b, c))", "a or Kernel.||(b, c)"], + ); + assert_group( + "a and (b or c)", + [3, 3, 3, 2], + &["Kernel.and(a, Kernel.or(b, c))"], + ); + assert_group( + "if a and b, do: 1", + [3, 3, 2, 2], + &["if Kernel.and(a, b), do: 1"], + ); +} + +/// A negation decides nothing in any spelling, applied or called on +/// `Kernel`; a user module's `and/2` is an ordinary function; and a +/// capture or a bare reference only names `Kernel`'s. +#[cfg(feature = "elixir")] +#[test] +fn elixir_kernel_negation_and_foreign_word_operator_decide_nothing() { + assert_group( + "not x", + [1, 1, 0, 0], + &["!x", "Kernel.not(x)", "Kernel.!(x)"], + ); + assert_group( + "Foo.foo(a, b)", + [1, 1, 0, 0], + &[ + "Foo.and(a, b)", + "Foo.or(a, b)", + "&Kernel.and/2", + "Kernel.and", + ], + ); +} + /// A user module's `||/2` is an ordinary function, and a capture only /// names `Kernel`'s: neither decides anything in any metric. #[cfg(feature = "elixir")] From 5f96b658d9c0c26a2b939f35ad9816891988e40f Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:17:07 -0700 Subject: [PATCH 20/34] docs(elixir): note why the clause gate ignores its owner `f(x, rescue: (e -> e))` counts its clause as a decision because the section gate does not ask which call owns the pair. The compiler rejects that argument for any function, so only a user macro can take it, and its clauses mean whatever the macro makes of them. Gating on the owner would move no valid program's score in a defensible direction, so the gap is documented rather than fixed and is not pinned by a test, per grammar-dispatch section 6. Also repoint the paragraph's stale `elixir_is_section_pair` link. Refs #1583 --- src/metrics/elixir_decisions.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/metrics/elixir_decisions.rs b/src/metrics/elixir_decisions.rs index 75e1213c..2c58197c 100644 --- a/src/metrics/elixir_decisions.rs +++ b/src/metrics/elixir_decisions.rs @@ -357,8 +357,20 @@ fn elixir_is_first_clause<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> /// [`elixir_when_is_guard`] excludes a typespec's `when` (#1531). The /// keyword form of a clause construct (`case(x, do: (1 -> :a; …))`) /// holds its clauses in the same parenthesised `block`, so a `block` -/// counts when it is the value of a section pair -/// ([`elixir_is_section_pair`]). +/// counts when it is the value of a pair whose key names a clause +/// section ([`DECISION_SECTION_KEYS`]). +/// +/// That pair is not asked which Call owns it, so a clause block passed +/// under a section key to *any* call counts: `f(x, rescue: (e -> e))` +/// scores one decision (#1583). Only a macro can take such an argument +/// — the compiler rejects it for a function ("cannot handle clauses with +/// the -> operator because it is not a macro", Elixir 1.17) — and a +/// user macro's clauses mean whatever it makes of them, so gating on the +/// owner would change no valid program's score in a way either answer +/// could defend. A dispatching DSL is the likelier reading, and is the +/// one kept. Cognitive gates its keyword sections on the owner for a +/// different reason: it scores the pair whatever its value holds, and +/// `else:` is an ordinary option to any function (`f(x, else: 1)`). /// /// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode`, which /// both score a clause construct per clause and must agree on which From 5b683843a06429f085057662445e9aad34a6c2a0 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:19:00 -0700 Subject: [PATCH 21/34] fix(elixir): apply review findings for #1581-#1583 The `Kernel` operator lookup repeated the `and` / `or` byte list that `elixir_short_circuit_name` owns; it now asks that helper, so the two cannot drift. The book still listed `after:` among the keys whose clauses score, which #1581 removed. --- big-code-analysis-ast/src/lang_helpers/elixir.rs | 5 +---- big-code-analysis-book/src/metrics.md | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/big-code-analysis-ast/src/lang_helpers/elixir.rs b/big-code-analysis-ast/src/lang_helpers/elixir.rs index 307f546f..f173bf2f 100644 --- a/big-code-analysis-ast/src/lang_helpers/elixir.rs +++ b/big-code-analysis-ast/src/lang_helpers/elixir.rs @@ -156,10 +156,7 @@ pub fn elixir_kernel_operator_token<'a>(call: &Node<'a>, code: &[u8]) -> Option< return name.child(0); } (name.kind_id() == Elixir::Identifier as u16 - && matches!( - code.get(name.start_byte()..name.end_byte()), - Some(b"and" | b"or") - )) + && elixir_short_circuit_name(&name, code).is_some()) .then_some(name) } diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 28374a12..00b94765 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -158,7 +158,7 @@ application would over-count. | Kotlin | A subject-less `when` arm scores through the predicate slot; a subject-ful arm scores per entry | A subject-less arm's condition (`when { x > 5 -> … }`) is an ordinary boolean expression, compiled as an `if` predicate, so the comparison inside it is already counted by the operator arms and a blanket per-entry increment double-counted it. A subject-ful arm (`when (x) { in 1..2 -> … }`) lists a pattern rather than an independent boolean expression, so the implicit `subject == pattern` is a decision nothing in the source spells and the entry itself pays for it (#1421). | | Ruby | A subject-less `case` scores each `when` pattern as an `\|\|` operand; a subject-ful `case` scores per clause | A subject-less clause (`case; when x > 5`) lists ordinary boolean expressions, evaluated like an `if` predicate, and several patterns (`when a, b`) are an implicit `\|\|`. Each pattern therefore scores once unless its own operator already counted it, so `when x > 5` scores 1 like `if x > 5`, and `when a, b` scores 2 like `if a \|\| b`. Before this, a blanket per-clause increment counted a comparison twice. A subject-ful clause (`case x; when 1`) matches each pattern with an implicit `pattern === x` that nothing in the source spells, so the clause pays once (#1453). | | Go | A tagless `switch` scores each case expression as an `\|\|` operand; a tagged `switch` scores per case | A tagless switch (`switch { case x > 5: }`, with or without an init statement) is `switch true`: each case expression is an ordinary boolean scored by the same slot an `if` predicate uses, and a list (`case a, b:`) is an implicit `\|\|`. So `case x > 5:` scores 1 like `if x > 5`, and `case a, b:` scores 2 like `if a \|\| b`. Before this, a blanket per-case increment counted a comparison twice. A tagged case (`switch x { case 1: }`) compares with an implicit `x == 1` that nothing in the source spells, so the case pays once; type-switch and `select` cases are unchanged (#1523). | -| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive`, of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). A bracketed keyword list in last position (`if(c, [do: 1, else: 2])`) is the same argument as the bare one and scores the same, as does a quoted key (`"else": 2`); a list anywhere else, or one with a non-pair item, holds no sections. A `Kernel.&&` / `Kernel.\|\|` remote call scores its operands as `a && b` / `a \|\| b` does, the piped value of `a \|> Kernel.\|\|(b)` included (#1574), and so do `Kernel.and` / `Kernel.or` (#1582). A `receive`'s `after` clause is its timeout, the path taken when no message clause matched in time, so it is the construct's default arm and scores nothing, as `_ ->` does in a `case` (#1581). | +| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive`, of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). A bracketed keyword list in last position (`if(c, [do: 1, else: 2])`) is the same argument as the bare one and scores the same, as does a quoted key (`"else": 2`); a list anywhere else, or one with a non-pair item, holds no sections. A `Kernel.&&` / `Kernel.\|\|` remote call scores its operands as `a && b` / `a \|\| b` does, the piped value of `a \|> Kernel.\|\|(b)` included (#1574), and so do `Kernel.and` / `Kernel.or` (#1582). A `receive`'s `after` clause is its timeout, the path taken when no message clause matched in time, so it is the construct's default arm and scores nothing, as `_ ->` does in a `case` (#1581). | | C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these brackets as an operator (a type-enclosing pair as one `<>` since #1559 and #1567), and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | | C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | From 9d4cd0f1081b33f890935f1a667db7eb3248e537 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:36:52 -0700 Subject: [PATCH 22/34] fix(perl): skip an auto-quoted die/exit key in nexits tree-sitter-perl 1.1.2 parses an auto-quoted hash key as a bareword call (upstream #1571), so `(exit => 1, die => 2)` scored two exits its quoted twin does not. ABC and `find --type call` already skip such a key through perl_bareword_is_key, which reads the token after the word. Exit::compute now takes the walk's Ancestors chain, as every other metric's compute does, and the Perl impl asks the key test after its text test matches. The chain carries the walk's SiblingCache, so a flat ERROR tail of keys stays linear. Exit is pub(crate), so the widened signature changes no public API. Fixes #1585 --- .bca-baseline.toml | 2 +- big-code-analysis-book/src/metrics.md | 2 +- src/macros/mod.rs | 10 +- src/metrics/nexits.rs | 217 +++++++++++++++++++++++--- src/spaces/compute.rs | 2 +- 5 files changed, 200 insertions(+), 33 deletions(-) diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 9945e733..410b2932 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -1179,7 +1179,7 @@ value = 5.0 path = "src/spaces/compute.rs" qualified = "compute_per_node" metric = "halstead.effort" -value = 63645.7536117776 +value = 64334.51855807396 [[entry]] path = "src/spaces/compute.rs" diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 00b94765..cd8ab8ce 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -163,7 +163,7 @@ application would over-count. | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | | C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | | Perl | A comparison's right operand is not a condition, although the grammar parses it as one | tree-sitter-perl 1.1.2 lets a chaining comparison (`<`, `>`, `<=`, `>=`, `lt`, `gt`, `le`, `ge`, `==`, `!=`, `eq`, `ne`) take everything of lower precedence to its right as its operand, so `$a > $b ? 1 : 2` parses as `$a > ($b ? 1 : 2)` and `$a > $b && $c` as `$a > ($b && $c)`. The ternary's condition slot and the chain's first operand then held `$b`, which scored as a condition of its own: one above the JavaScript twin for each misparsed comparison. The walker recognises the shape and leaves that operand to the comparison, so both spellings score what the parenthesised `($a > $b) ? 1 : 2` and `($a > $b) && $c` do. An explicit `$a > ($b ? 1 : 2)` is unaffected. `<=>`, `cmp`, `=~` and `!~` do not chain and parse correctly. The workaround goes when the grammar is fixed upstream (#1273). | -| Perl | An auto-quoted hash key is a string, not a call | Perl quotes a bareword before `=>` and alone in a hash subscript, a leading `-` included, so `(name => 1)`, `new(name => $x)`, `(-name => 1)` and `$h{-name}` hold the strings `"name"` and `"-name"`. tree-sitter-perl 1.1.2 parses each word as a call, which scored a branch, and each `-name` as a negation, which billed a Halstead `-` operator and the operand `name`. Each now scores as its quoted twin `('name' => 1)` does, and Halstead bills the key as one operand spelled as written: `name`, `-name`. A package-qualified word (`Foo::Bar => 1`), which `=>` does not quote, a word with arguments (`name() => 1`) and a bareword value (`(a => foo)`) keep their call. `bca find --type call` reports the same calls ABC counts (#1580). The workaround goes when the grammar is fixed upstream (#1571, #1572). | +| Perl | An auto-quoted hash key is a string, not a call | Perl quotes a bareword before `=>` and alone in a hash subscript, a leading `-` included, so `(name => 1)`, `new(name => $x)`, `(-name => 1)` and `$h{-name}` hold the strings `"name"` and `"-name"`. tree-sitter-perl 1.1.2 parses each word as a call, which scored a branch, and each `-name` as a negation, which billed a Halstead `-` operator and the operand `name`. Each now scores as its quoted twin `('name' => 1)` does, and Halstead bills the key as one operand spelled as written: `name`, `-name`. A package-qualified word (`Foo::Bar => 1`), which `=>` does not quote, a word with arguments (`name() => 1`) and a bareword value (`(a => foo)`) keep their call. `bca find --type call` reports the same calls ABC counts (#1580). Nor does `nexits` count a `die` or `exit` key as an exit (#1585). The workaround goes when the grammar is fixed upstream (#1571, #1572). | #### Where a constructor call lands {#abc-constructor-attribution} diff --git a/src/macros/mod.rs b/src/macros/mod.rs index c2c3efb5..603417d3 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -48,7 +48,7 @@ macro_rules! implement_metric_trait { ); // Internal helper: shared no-op body for traits whose `compute` // signature is `<'a>(&Node<'a>, &'a [u8], Ancestors<'a, '_>, - // &mut Stats)` (Abc, Cyclomatic). Public arms below delegate here + // &mut Stats)` (Abc, Cyclomatic, Exit). Public arms below delegate here // so the body is written once. `Npa` and `Npm` share the signature // but need `HAS_MEMBERS = false` as well, so they route through // `@code_and_chain_taking_memberless` instead — reaching for this @@ -66,14 +66,8 @@ macro_rules! implement_metric_trait { } )+ ); - // `Exit` is the one metric whose `compute` still takes no ancestor - // chain: no language's exit rule asks what encloses the node. (Exit, $($code:ident),+) => ( - $( - impl Exit for $code { - fn compute<'a>(_node: &Node<'a>, _code: &'a [u8], _stats: &mut Stats) {} - } - )+ + implement_metric_trait!(@code_and_chain_taking Exit, $($code),+); ); (Cyclomatic, $($code:ident),+) => ( implement_metric_trait!(@code_and_chain_taking Cyclomatic, $($code),+); diff --git a/src/metrics/nexits.rs b/src/metrics/nexits.rs index 347c1a49..8716a4a5 100644 --- a/src/metrics/nexits.rs +++ b/src/metrics/nexits.rs @@ -20,6 +20,7 @@ use std::fmt; use crate::checker::Checker; +use crate::lang_helpers::perl::perl_bareword_is_key; use crate::macros::implement_metric_trait; use crate::*; @@ -136,7 +137,16 @@ where { /// Walk `node` and update `stats` with this metric for the language /// implementing the trait. - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats); + /// + /// `ancestors` is `node`'s chain as the metric walk recorded it, + /// sibling cache included; Perl reads it to tell an auto-quoted + /// `die =>` key from the builtin (#1585). + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ); } // Bumps `stats.exit` whenever the current node matches any of the @@ -145,7 +155,12 @@ where macro_rules! impl_exit_match_kinds { ($code:ty, $lang:ident, [$($kind:ident),+ $(,)?]) => { impl Exit for $code { - fn compute<'a>(node: &Node<'a>, _code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + _code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if matches!(node.kind_id().into(), $($lang::$kind)|+) { stats.exit += 1; } @@ -229,7 +244,12 @@ impl_exit_match_kinds!( ); impl Exit for RustCode { - fn compute<'a>(node: &Node<'a>, _code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + _code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { // Count only explicit `return` and `?` (TryExpression). The // implicit final-expression path is NOT an exit — peer-language // impls have the same convention. See #243 for the prior bug @@ -245,7 +265,12 @@ impl Exit for RustCode { } impl Exit for CsharpCode { - fn compute<'a>(node: &Node<'a>, _code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + _code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if matches!( node.kind_id().into(), Csharp::ReturnStatement @@ -267,7 +292,12 @@ impl Exit for GoCode { // `selector_expression`) means a package-qualified call like // `foo.panic()` — a user function, not the built-in — is not counted, // mirroring how Bash matches the bare builtin command name. - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if matches!(node.kind_id().into(), Go::ReturnStatement) { stats.exit += 1; } else if node.kind_id() == Go::CallExpression @@ -307,19 +337,23 @@ impl Exit for PerlCode { // plain `identifier`, never a bareword, so a user method named // `die` is not counted. // - // Known artifact: a fat-comma hash key auto-quotes its bareword in - // Perl, but the grammar still emits `call_expression_with_bareword` - // for it, so `(die => 1)` counts as an exit. Gating on the `=>` - // that follows would need a forward sibling lookup from inside the - // metric walk, which #1096 took out of these bodies; the same node - // already counts as an ABC branch (`src/metrics/abc/perl.rs`), so - // the artifact is accepted rather than paid for here. - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + // - An auto-quoted key (`(die => 1)`) is a string, though the + // grammar parses it as a bareword call (#1571 upstream). ABC and + // `find --type call` skip it, and so does this (#1585). The text + // test runs first: the key test reads a sibling, which the walk's + // `SiblingCache` keeps linear under a flat `ERROR`. + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { let is_abrupt_exit_builtin = node.kind_id() == Perl::CallExpressionWithBareword && matches!( node.utf8_text(code), Some("die" | "exit" | "CORE::die" | "CORE::exit") - ); + ) + && !perl_bareword_is_key(node, ancestors); if node.kind_id() == Perl::ReturnExpression || is_abrupt_exit_builtin { stats.exit += 1; } @@ -327,7 +361,12 @@ impl Exit for PerlCode { } impl Exit for KotlinCode { - fn compute<'a>(node: &Node<'a>, _code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + _code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if matches!( node.kind_id().into(), Kotlin::ReturnExpression | Kotlin::ThrowExpression @@ -346,7 +385,12 @@ impl Exit for LuaCode { // `os.exit`. Count them as exits alongside `return`. Matching the exact // callee text means a user call such as `foo()` or `myError()` is not // counted, mirroring how Bash/Elixir match the bare builtin name. - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if node.kind_id() == Lua::ReturnStatement { stats.exit += 1; } else if node.kind_id() == Lua::FunctionCall @@ -359,7 +403,12 @@ impl Exit for LuaCode { } impl Exit for BashCode { - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { // Bash has no `return_statement` node: `return` and `exit` are // ordinary builtins parsed as `Bash::Command` whose `name` field // points at a `Bash::CommandName`. Identify them by comparing the @@ -374,7 +423,12 @@ impl Exit for BashCode { } impl Exit for TclCode { - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { // Tcl has no return keyword node; `return` is a generic Command whose // name field is a simple_word with text "return" — the same // leading-word resolution the Cognitive / Cyclomatic `switch` and @@ -399,7 +453,12 @@ impl Exit for TclCode { } impl Exit for IrulesCode { - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { // Like Tcl, iRules has no `return` keyword node — `return` is a // generic Command (it is not among the grammar's `_builtin` // commands). The bare name word can surface as either `simple_word` @@ -441,7 +500,12 @@ impl Exit for PhpCode { // text — brittle and likely to false-match user-defined `die` // functions. Modern PHP idiom favors `throw new Exception()` over // `die`, so leaving this asymmetric is acceptable. - fn compute<'a>(node: &Node<'a>, _code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + _code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if matches!( node.kind_id().into(), Php::ReturnStatement | Php::YieldExpression | Php::ThrowExpression | Php::ExitStatement @@ -492,7 +556,12 @@ impl Exit for RubyCode { // The callee is inspected before the receiver: every call has a // `method`, and the text test rejects nearly all of them, so the // receiver lookup runs only for the handful that spell a builtin. - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if matches!(node.kind_id().into(), Ruby::Return | Ruby::Return2) { stats.exit += 1; } else if Self::is_call(node) @@ -514,7 +583,12 @@ impl Exit for ElixirCode { // `raise`, `reraise`, or `exit`, all of which surface as `Call` // nodes whose target is an `Identifier` whose text spells the // keyword. Mirrors the Bash/Tcl pattern of comparing target text. - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if node.kind_id() == Elixir::Call && let Some(target) = node.child_by_field_name("target") && target.kind_id() == Elixir::Identifier @@ -1602,6 +1676,105 @@ mod tests { ); } + /// The root space's `nexits` sum for the Perl file `source`. + #[cfg(feature = "perl")] + fn perl_nexits(source: &str) -> u64 { + let ast = Ast::parse(Source::new(LANG::Perl, source.as_bytes())).expect("perl is enabled"); + ast.metrics(MetricsOptions::default()) + .expect("metrics yields a root space") + .metrics + .nexits + .nexits_sum() + } + + /// Each auto-quoted `die` / `exit` key scores exactly as its quoted + /// twin does (#1585). tree-sitter-perl reads `die` before `=>` as a + /// bareword call (#1571 upstream), so the `=>` rows each counted one + /// exit per key before the fix: the first row read 3. The subscript + /// and dash rows never parse to a `die` / `exit` bareword at all + /// (`$h{die}` is an `identifier`, `-exit` a file test on `xit`); they + /// pin that the gate keeps them at the twin's reading. `- die =>` + /// negates the quoted string, and a newline before the `=>` does not + /// stop it quoting. + #[cfg(feature = "perl")] + #[test] + fn perl_exit_key_scores_like_its_quoted_twin() { + let rows: [(&str, u64); 9] = [ + ( + "sub f { my %h = ('exit' => 1, 'die' => 2); return 1; }\n", + 1, + ), + ("sub f { my %h = (- 'die' => 1); return 1; }\n", 1), + ("sub f { my %h = ('exit'\n => 1); return 1; }\n", 1), + ( + "sub f { my $r = { 'die' => 1, 'exit' => 2 }; return $r; }\n", + 1, + ), + ("sub f { g('exit' => 1); return; }\n", 1), + ( + "sub f { my %h = ('-die' => 1, '-exit' => 2); return 1; }\n", + 1, + ), + ("sub f { my $v = $h{'die'}; return $v; }\n", 1), + ("sub f { my $v = $h{'-exit'}; return $v; }\n", 1), + ("sub f { my %h = ('exit' => 1); exit 2; }\n", 1), + ]; + for (twin, want) in rows { + let key = twin.replace('\'', ""); + assert_eq!(perl_nexits(twin), want, "`{twin}` moved; re-derive the row"); + assert_eq!(perl_nexits(&key), want, "`{key}` must score like `{twin}`"); + } + } + + /// A `die` / `exit` that is no auto-quoted key keeps its exit: every + /// call spelling, a call inside a subscript, a call before `=>` + /// (`die()` takes arguments, so `=>` does not quote it), a value + /// after `=>`, and the `CORE::` spelling, which `=>` does not quote + /// either since `::` is no word character. Every value is `main`'s, + /// measured before #1585. + #[cfg(feature = "perl")] + #[test] + fn perl_exit_outside_a_key_keeps_its_exit() { + let rows: [(&str, u64); 9] = [ + ("sub f { die \"x\"; }\n", 1), + ("sub f { exit 1; }\n", 1), + ("sub f { exit; }\n", 1), + ("sub f { open(my $fh, '<', 'p') or die \"x\"; }\n", 1), + ("sub f { my $v = $h{die()}; }\n", 1), + ("sub f { my $v = $h{exit()}; }\n", 1), + ("sub f { my %h = (die() => 1); }\n", 1), + ("sub f { my %h = (x => die); }\n", 1), + ("sub f { my %h = (CORE::die => 1); }\n", 1), + ]; + for (source, want) in rows { + assert_eq!(perl_nexits(source), want, "`{source}` changed its reading"); + } + } + + /// A file whose tail fails to parse leaves every key and call in it a + /// direct child of one flat `ERROR` root. The key test reads the + /// token after each `die`, which the metric walk's `SiblingCache` + /// answers (#1584); a walk that lent the cache without recording the + /// `ERROR` would trip the debug assertion in that lookup. The reading + /// is the twelve `exit` calls, as the quoted twin's is. + #[cfg(feature = "perl")] + #[test] + fn perl_exit_key_in_a_flat_error_scores_like_its_twin() { + // Past `WIDE_PARENT` in `big-code-analysis-ast/src/node/ancestors.rs`, + // the width from which a walk records a node's children. + const RECORDED_WIDTH: usize = 64; + let key = format!("my @r = ({}", "{\ndie => 1,\nexit,\n".repeat(12)); + let twin = key.replace("die =>", "'die' =>"); + let ast = Ast::parse(Source::new(LANG::Perl, key.as_bytes())).expect("perl is enabled"); + let root = ast.as_tree_sitter().root_node(); + assert!( + root.is_error() && root.child_count() > RECORDED_WIDTH, + "the fixture must parse to a wide flat ERROR root" + ); + assert_eq!(perl_nexits(&twin), 12); + assert_eq!(perl_nexits(&key), 12); + } + #[cfg(feature = "typescript")] #[test] fn tsx_function_with_returns() { diff --git a/src/spaces/compute.rs b/src/spaces/compute.rs index fff1405e..51b6f3a1 100644 --- a/src/spaces/compute.rs +++ b/src/spaces/compute.rs @@ -428,7 +428,7 @@ fn compute_per_node<'a, T: MetricSuite>( T::NArgs::compute(node, code, ancestors, &mut last.metrics.nargs); } if selected.contains(Metric::Nexits) { - T::Exit::compute(node, code, &mut last.metrics.nexits); + T::Exit::compute(node, code, ancestors, &mut last.metrics.nexits); } if selected.contains(Metric::Abc) { T::Abc::compute(node, code, ancestors, &mut last.metrics.abc); From 1c884c91ed807a3fd84b776dd48de564b4531041 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:46:49 -0700 Subject: [PATCH 23/34] docs(changelog): record the batch follow-up fixes Add the Unreleased entries for #1572, #1574, #1575, #1578-#1582, #1584 and #1585. --- CHANGELOG.md | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81a57921..6980e734 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,17 @@ for historical reference. ## [Unreleased] +### Added + +- **Seven Perl and Bash scaling probes in the complexity-class gate** + (`make bench-scaling`, #1575): the Perl ABC misparse climb, Perl + ABC's `=~` binding check under an `ERROR` root, and the Bash + cognitive / ABC comparison-run walks (`abc/perl-or-chain`, + `abc/perl-misparsed-comparison`, `abc/perl-and-statements`, + `abc/perl-error-tail`, `cognitive/bash-test-connective-run`, + `abc/bash-arith-assign-chain`, `abc/bash-eq-and-test`). Each was + watched tripping with its pre-fix walk restored. + ### Changed - **Halstead bills angle brackets that enclose types as one `<>` @@ -90,6 +101,101 @@ for historical reference. format-spec `:`. The alignment value still counts: `8` is an operand, and `{x,-w}` counts `-` and `w`. A `,` or `:` inside the hole — a call argument list, ternary or named argument — still counts. +- **C, C++ and Mozcpp bill every storage class, function specifier and + type qualifier as a Halstead operator** (#1578): `static`, `extern`, + `register`, `inline`, `thread_local`, `const`, `volatile`, `restrict`, + `_Atomic`, `constexpr`, `_Noreturn`, `alignas`, their vendor spellings + (`__inline`, `__forceinline`, `__restrict__`, `__thread`, + `__extension__`, …) and C's storage class `auto`. C++ and Mozcpp also + bill `virtual`, `mutable`, `constinit`, `consteval`, `explicit`, + `friend`, the access labels, `override`, `final` and the `noexcept` + of an exception specification. Each counts once as its keyword, never + as the `storage_class_specifier` / `type_qualifier` / + `virtual_specifier` / `access_specifier` wrapper. `override` and + `final` used as names stay operands, `extern "C"` keeps `"C"` an + operand, and C++'s placeholder `auto`, attributes, MS calling + conventions and the Gecko `MOZ_*` macros are unchanged. Halstead and + MI scores of C-family code rise. +- **Objective-C bills declaration modifiers as Halstead operators** + (#1579). C's storage classes, function specifiers and type qualifiers + bill as in C, so a `.m` scores a C declaration the same as a `.c` or + `.mm`. ObjC's own modifiers also bill: the `@public` / `@private` / + `@protected` / `@package` ivar labels, `@optional` / `@required`, the + ARC ownership qualifiers and `__bridge` casts, the nullability + qualifiers, `__kindof`, `__covariant` / `__contravariant`, `_Complex`, + the `in` / `out` / `inout` / `bycopy` / `byref` / `oneway` method-type + qualifiers, and property attribute words (`nonatomic`, `copy`, + `getter`, …). A `getter=` / `setter=` value stays an operand. Apple + SDK macros (`NS_INLINE`, `FOUNDATION_EXPORT`, `IBOutlet`, `__unused`) + bill nothing. +- **Perl bareword hash keys no longer score as calls** (#1572). + tree-sitter-perl 1.1.2 parses an auto-quoted bareword key as a call, + so `(name => 1)`, every named argument (`new(name => $x, size => 3)`) + and `{ a => { b => 1 } }` scored one ABC branch per key, and + `(-name => 1)` / `$h{-name}` also billed a Halstead `-` operator and + the unsigned operand `name`. Each key now scores like its quoted twin + and is billed as written (`name`, `-name`), including a word that + ends a longer expression before `=>` (`(1 - name => 2)`), since `=>` + quotes the word right before it. Package-qualified words + (`Foo::Bar => 1`), words with arguments (`name() => 1`), bareword + values (`(a => foo)`) and `$h{+shift}` keep their call. Cyclomatic and + cognitive are unchanged. This is a local workaround for upstream + #1571. +- **Perl `bca find --type call` and `bca count --type call` report each + call once and skip auto-quoted keys** (#1580). tree-sitter-perl parses + a call's name (`foo` in `foo(1)`, `print` in `print "x"`) as a bareword + call nested inside the call, and parses `(name => 1)`, `(-size => 2)` + and `$h{-name}` as calls too; the issue's fixture went from five + matches to one. The filter now applies the gate ABC already used, so a + Perl file's `call` count equals its `abc.branches`. A bareword with no + wrapper still counts: `foo;`, `&foo;`, and the wrapper-less `foo(1` + that error recovery leaves. New + `Checker::is_call_with_code(node, code, ancestors)` in + `big-code-analysis-ast`, forwarding to `is_call` by default. +- **Perl `nexits` no longer counts an auto-quoted `die` or `exit` hash + key as an exit point** (#1585): `(exit => 1, die => 2)` now scores as + its quoted twin does. A real `die` / `exit` call, including + `$h{die()}`, `die() => 1`, `(x => die)` and `CORE::die => 1`, still + counts. The crate-private `Exit::compute` now takes the ancestor + chain; no public API changes. +- **Elixir scores every spelling of a construct alike** (#1574). A + trailing bracketed keyword list (`if(c, [do: 1, else: 2])`, + `for(x <- xs, [reduce: 0, do: …])`) and a quoted key (`"else": 2`) now + score exactly as the bare keyword form and the `do … end` block do. + Before, the bracketed `with` lost its modified-cyclomatic `else` tier, + the bracketed `for … reduce:` scored its accumulator head as a + decision, the bracketed `cond` lost its free `true ->`, and the + bracketed `if` / `unless` lost ABC's `else` condition. + `Kernel.&&(a, b)` / `Kernel.||(a, b)`, including the piped + `a |> Kernel.||(b)`, now score ABC conditions and a cognitive boolean + sequence as `a && b` / `a || b` do; cyclomatic already counted them, + and a user module's `Foo.||(a, b)` still scores nothing. Cognitive now + scores the keyword forms' `else:` (+1) and `rescue:` / `catch:` + (+nesting) for `if`, `unless`, `try` and the `def` family, as their + blocks do; the keyword `try` scored 0 before. A `with`'s `else` adds + no cognitive cost in either spelling: its clauses are the arms of a + `case`, already paid by the container (the block form scored 2 + against the keyword form's 1). +- **Elixir `receive` scores like `case`** (#1581): modified cyclomatic + counts the construct once and cognitive complexity adds +nesting, in + the block and keyword forms. Its `after` timeout clause is the + construct's default arm and no longer scores in standard cyclomatic or + ABC, and a `receive` holding only an `after` scores nothing. +- **Elixir `Kernel.and(a, b)` and `Kernel.or(a, b)` count as + short-circuit decisions** in cyclomatic, ABC conditions and cognitive + complexity (#1582), matching `a and b` / `a or b` and the `Kernel.&&` + / `Kernel.||` calls. + +### Performance + +- **Perl ABC and Halstead no longer go quadratic on a file whose tail + fails to parse into a flat `ERROR` node** (#1584). The bareword-key, + dash-key, key-sign and misparse-operator lookups now read the children + the walk recorded (a `SiblingCache` lent through `Ancestors`) instead + of seeking from the `ERROR`'s start: 16 000 copies of `{ last,` take + 0.03 s, down from 2.2 s. The same change makes Kotlin Halstead linear + on a string of many short `$name` interpolations, which was quadratic + on valid input (8 000 names: 11.5 s → 0.01 s). ## [2.3.1] - 2026-10-10 From e2978ec6b8d32b04191acdc19474654e6d848037 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:06:20 -0700 Subject: [PATCH 24/34] fix(elixir): give receive one free arm, review items B1: a `receive` now gets exactly one free arm: its bare `_ ->` if the do-section has one, otherwise its `after` timeout. Beside a catch-all the timeout is a third path, so `receive do {:a, y} -> y; _ -> 0 after 10 -> :t end` scores what the three-clause `case` does in standard cyclomatic and ABC; modified and cognitive pay the container only and are unchanged. New parity rows in all three spellings. T2: `WIDE_PARENT` is `pub` + `#[doc(hidden)]` and the three wide-parent fixtures size themselves from it instead of a copied 64. T3: note that `act_on_node_records_a_wide_seeded_ancestor` discriminates only in a debug build (lesson #80). T5: `nexits/perl-error-tail-keys` scaling probe (1.11 clean, 1.87 with `Ancestors::lookup` ignoring the cache), listed in benchmarking.md. T4/D2: C++ and C both parse `int restrict = 0;` / `int noreturn = 1;` as a qualifier before a zero-width name; documented as grammar-decided in `getter/c.rs`. D1, D3, N1: doc, changelog and comment fixes. --- CHANGELOG.md | 28 ++++--- big-code-analysis-ast/src/getter/c.rs | 9 ++- .../src/lang_helpers/elixir.rs | 4 +- big-code-analysis-ast/src/node.rs | 11 ++- big-code-analysis-ast/src/node/ancestors.rs | 13 +++- big-code-analysis-bench/src/shapes.rs | 34 +++++++++ docs/development/benchmarking.md | 7 +- src/metrics/abc.rs | 5 +- src/metrics/cognitive/elixir.rs | 3 +- src/metrics/cyclomatic/elixir.rs | 5 +- src/metrics/elixir_decisions.rs | 74 ++++++++++++++++--- src/metrics/elixir_spelling_parity_tests.rs | 19 ++++- src/metrics/nexits.rs | 7 +- src/metrics/perl_keyword_key_tests.rs | 7 +- 14 files changed, 177 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6980e734..6fb05692 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,14 +26,19 @@ for historical reference. ### Added -- **Seven Perl and Bash scaling probes in the complexity-class gate** - (`make bench-scaling`, #1575): the Perl ABC misparse climb, Perl - ABC's `=~` binding check under an `ERROR` root, and the Bash - cognitive / ABC comparison-run walks (`abc/perl-or-chain`, - `abc/perl-misparsed-comparison`, `abc/perl-and-statements`, - `abc/perl-error-tail`, `cognitive/bash-test-connective-run`, - `abc/bash-arith-assign-chain`, `abc/bash-eq-and-test`). Each was - watched tripping with its pre-fix walk restored. +- **Twelve Perl, Bash and Kotlin scaling probes in the + complexity-class gate** (`make bench-scaling`). #1575 adds seven: the + Perl ABC misparse climb, Perl ABC's `=~` binding check under an + `ERROR` root, and the Bash cognitive / ABC comparison-run walks + (`abc/perl-or-chain`, `abc/perl-misparsed-comparison`, + `abc/perl-and-statements`, `abc/perl-error-tail`, + `cognitive/bash-test-connective-run`, `abc/bash-arith-assign-chain`, + `abc/bash-eq-and-test`). The rest price sibling lookups: + `abc/perl-bareword-keys` (#1572), `halstead/perl-error-tail-keys` + and `halstead/kotlin-short-interpolations` (#1584), + `count/perl-error-tail-calls` (#1580) and + `nexits/perl-error-tail-keys` (#1585). Each was watched tripping with + its pre-fix walk restored. ### Changed @@ -178,9 +183,10 @@ for historical reference. against the keyword form's 1). - **Elixir `receive` scores like `case`** (#1581): modified cyclomatic counts the construct once and cognitive complexity adds +nesting, in - the block and keyword forms. Its `after` timeout clause is the - construct's default arm and no longer scores in standard cyclomatic or - ABC, and a `receive` holding only an `after` scores nothing. + the block and keyword forms. A `receive` gets one free arm, as a + `case` does: its bare `_ ->` if it has one, otherwise its `after` + timeout clause, which then no longer scores in standard cyclomatic or + ABC. A `receive` holding only an `after` scores nothing. - **Elixir `Kernel.and(a, b)` and `Kernel.or(a, b)` count as short-circuit decisions** in cyclomatic, ABC conditions and cognitive complexity (#1582), matching `a and b` / `a or b` and the `Kernel.&&` diff --git a/big-code-analysis-ast/src/getter/c.rs b/big-code-analysis-ast/src/getter/c.rs index 2a2009a3..eaf3876f 100644 --- a/big-code-analysis-ast/src/getter/c.rs +++ b/big-code-analysis-ast/src/getter/c.rs @@ -100,7 +100,14 @@ impl Getter for CCode { // where it is the prefix of an `extension_expression`. // Attributes (`__attribute__`, `__declspec`, `[[…]]`) and // the MS calling conventions stay unbilled: they annotate a - // declaration rather than spell part of it. + // declaration rather than spell part of it. `cpp.rs` copies + // this list, though `restrict` and `noreturn` are no C++ + // keywords: `int restrict = 0;` is a valid C++ variable, as + // `int noreturn = 1;` is in C without ``. Both + // grammars still parse the name as the qualifier before a + // zero-width `identifier`, so the keyword bills and no + // operand does — the grammar decides that, not this arm, as + // `objc.rs` notes for `nullable`. | Static | Extern | Register | Auto | Inline | Inline2 | Inline3 | Forceinline | ThreadLocal | Thread | Const | Constexpr | Volatile | Restrict | Restrict2 | Atomic | Noreturn | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension diff --git a/big-code-analysis-ast/src/lang_helpers/elixir.rs b/big-code-analysis-ast/src/lang_helpers/elixir.rs index f173bf2f..b0667418 100644 --- a/big-code-analysis-ast/src/lang_helpers/elixir.rs +++ b/big-code-analysis-ast/src/lang_helpers/elixir.rs @@ -135,8 +135,8 @@ pub fn elixir_applying_operator<'a>( /// function or decides nothing, and is not found. Nor is the childless /// `..`. /// -/// Matched by rule name: `call`, `dot` and `arguments` are aliased to -/// several ids each (grammar-dispatch §1). +/// Matched by rule name: `dot` and `arguments` are aliased to several +/// ids each (grammar-dispatch §1). #[must_use] pub fn elixir_kernel_operator_token<'a>(call: &Node<'a>, code: &[u8]) -> Option> { if call.kind() != "call" || !call.children().any(|child| child.kind() == "arguments") { diff --git a/big-code-analysis-ast/src/node.rs b/big-code-analysis-ast/src/node.rs index 877451df..669bc3f1 100644 --- a/big-code-analysis-ast/src/node.rs +++ b/big-code-analysis-ast/src/node.rs @@ -15,6 +15,8 @@ mod ancestors; mod parser_cache; +#[doc(hidden)] +pub use ancestors::WIDE_PARENT; pub use ancestors::{AncestorIter, Ancestors, SiblingCache}; use tree_sitter::Node as OtherNode; @@ -1886,12 +1888,13 @@ mod tests { #[cfg(feature = "mozjs")] #[test] fn act_on_node_records_a_wide_seeded_ancestor() { - // Past `WIDE_PARENT` in `node/ancestors.rs`. - const RECORDED_WIDTH: usize = 64; - let code = "var a = 1;\n".repeat(RECORDED_WIDTH + 6); + // Discriminates only through `Ancestors::lookup`'s `debug_assert!`: + // meaningful under `make pre-commit`'s debug build, vacuous under + // `--release` (lesson #80). + let code = "var a = 1;\n".repeat(WIDE_PARENT + 6); let tree = Tree::new::(code.as_bytes()); let root = tree.get_root(); - assert!(root.child_count() > RECORDED_WIDTH, "the root must be wide"); + assert!(root.child_count() > WIDE_PARENT, "the root must be wide"); let subtree = root.child(3).expect("fixture has a fourth statement"); let mut after = None; diff --git a/big-code-analysis-ast/src/node/ancestors.rs b/big-code-analysis-ast/src/node/ancestors.rs index 01b28608..9ce6d95b 100644 --- a/big-code-analysis-ast/src/node/ancestors.rs +++ b/big-code-analysis-ast/src/node/ancestors.rs @@ -277,10 +277,15 @@ enum Lookup<'tree, 'chain> { Narrow(Option>), } -// Children past which a walk records a node's children for sibling -// lookups. A seek over this many is a short scan however the grammar -// stores them, so recording a narrower node would copy it for nothing. -const WIDE_PARENT: usize = 64; +/// Children past which a walk records a node's children for sibling +/// lookups. A seek over this many is a short scan however the grammar +/// stores them, so recording a narrower node would copy it for nothing. +/// +/// `pub` and `#[doc(hidden)]` only so the wide-parent fixtures in this +/// crate's and `big-code-analysis`'s tests size themselves from it: a +/// hand-copied width would silently go narrow if this were raised. +#[doc(hidden)] +pub const WIDE_PARENT: usize = 64; // `node`'s index in `children`, a parent's children in order. Their start // bytes never decrease, so a binary search lands on the first child diff --git a/big-code-analysis-bench/src/shapes.rs b/big-code-analysis-bench/src/shapes.rs index ebdbe7e7..6e846445 100644 --- a/big-code-analysis-bench/src/shapes.rs +++ b/big-code-analysis-bench/src/shapes.rs @@ -651,6 +651,20 @@ pub fn perl_error_tail_calls(width: usize) -> String { format!("my @r = ({}", "{\nlast,\nname => 1,\n".repeat(width)) } +/// Perl: `my @r = (` followed by `width` copies of +/// `{⏎die => 1,⏎exit => 1,⏎exit,⏎`, never closed. +/// +/// [`perl_error_tail`]'s flat `ERROR`, holding the `die` / `exit` keys +/// nexits asks about: whether a `=>` follows each, which makes it a key +/// rather than an exit. The bare `exit` is the one exit per copy. +#[must_use] +pub fn perl_error_tail_exit_keys(width: usize) -> String { + format!( + "my @r = ({}", + "{\ndie => 1,\nexit => 1,\nexit,\n".repeat(width) + ) +} + /// Bash: `if [ "$a" = 0 -o "$a" = 0 -a … ]; then :; fi`, the /// connectives alternating. /// @@ -1688,6 +1702,25 @@ pub const PROBES: &[Probe] = &[ quadratic in the tail. The reading is three operands \ per copy of the body.", }, + Probe { + name: "nexits/perl-error-tail-keys", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Nexits], + reading: |m| m.nexits.nexits_sum(), + }, + render: perl_error_tail_exit_keys, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "`abc/perl-error-tail`'s flat `ERROR`, holding the \ + `die` and `exit` keys nexits must not score: the key \ + test asks for the token after each (#1585), which \ + the walk's `SiblingCache` answers; read off the \ + `ERROR` from its start it would be quadratic in the \ + tail. The reading is one `exit` per copy of the body.", + }, Probe { name: "count/perl-error-tail-calls", lang: LANG::Perl, @@ -1927,6 +1960,7 @@ mod tests { const RECOVERY_PROBES: &[&str] = &[ "abc/perl-error-tail", "halstead/perl-error-tail-keys", + "nexits/perl-error-tail-keys", "count/perl-error-tail-calls", ]; diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index e5a89119..ab52fe4c 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -169,6 +169,7 @@ Read it as follows. | `abc/perl-and-statements` | Perl | width | the misparse check's operator lookup in a block (#1273) | linear | | `abc/perl-error-tail` | Perl | width | the binding, bareword-key and misparse lookups under a flat `ERROR` root (#1540, #1584) | linear | | `halstead/perl-error-tail-keys` | Perl | width | the key and sign lookups under the same `ERROR` (#1584) | linear | +| `nexits/perl-error-tail-keys` | Perl | width | nexits' `die` / `exit` key lookup under the same `ERROR` (#1585) | linear | | `count/perl-error-tail-calls` | Perl | width | the `call` filter's bareword-key lookup under the same `ERROR`, on the `find` / `count` walk (#1580) | linear | | `cognitive/bash-test-connective-run` | Bash | depth | the comparison-run end of a `[ … -o … ]` test (#1536) | linear | | `abc/bash-arith-assign-chain` | Bash | depth | ABC's `=` comparison test on `(( a = … ))` (#1536) | linear | @@ -409,14 +410,16 @@ children of every node wider than 64 as they push them, in a array instead of seeking. `Ancestors::previous_sibling` reads it too: it scanned its parent from the start, which made a valid Kotlin string of `$name` interpolations quadratic (11.5 s for 8 000 names). With -`Ancestors::lookup` patched to ignore the cache, these three probes go -over bound and nothing else moves: +`Ancestors::lookup` patched to ignore the cache, these four probes go +over bound, as does `count/perl-error-tail-calls` below, whose walk +reads the same cache: | Walk put back | Probe that fails | Clean | Regressed | Top rung | |---|---|---|---|---| | sibling lookups seek in the flat `ERROR` | `abc/perl-error-tail` | 1.08 | 2.10 | 8.8 ms → 326 ms | | (same patch) | `halstead/perl-error-tail-keys` | 1.04 | 1.74 | 14.5 ms → 96.0 ms | | (same patch) | `halstead/kotlin-short-interpolations` | 0.96 | 2.02 | 1.1 ms → 589 ms | +| (same patch) | `nexits/perl-error-tail-keys` | 1.11 | 1.87 | 5.5 ms → 127 ms | The walk `bca find` and `bca count` share, `act_on_node`, keeps the same chain and lent no cache until Perl's `call` filter started asking the diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index e42ef0f9..333b3059 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -11718,9 +11718,8 @@ end // clause slot pays for, `catch_two` a multi-pattern clause, `recv` a // `receive`'s `after` timeout clause — free since #1581, as the // default arm, so `recv` scores what `case2` does — and `with_else` - // the `else` of - // a `with`, whose clauses pay rather than a Rule 5 `else`. Its `<-` - // clause pays one more since #1535, in both metrics. + // the `else` of a `with`, whose clauses pay rather than a Rule 5 + // `else`. Its `<-` clause pays one more since #1535, in both metrics. #[cfg(feature = "elixir")] #[test] fn elixir_clause_constructs_pay_per_clause() { diff --git a/src/metrics/cognitive/elixir.rs b/src/metrics/cognitive/elixir.rs index 6a0e7bc0..df715dd2 100644 --- a/src/metrics/cognitive/elixir.rs +++ b/src/metrics/cognitive/elixir.rs @@ -146,7 +146,8 @@ impl Cognitive for ElixirCode { // it has message clauses; one holding only an `after` // timeout is a timed wait, the idiomatic sleep, and // branches on nothing. The `after` adds nothing either - // way: it is the default arm, which `case` does not pay. + // way: like `case`, the container pays and its arms do + // not, a timeout beside a bare `_ ->` included. Some("receive") if elixir_receive_dispatches(node, code) => { increase_nesting(stats, &mut nesting); } diff --git a/src/metrics/cyclomatic/elixir.rs b/src/metrics/cyclomatic/elixir.rs index 7a3af483..82751428 100644 --- a/src/metrics/cyclomatic/elixir.rs +++ b/src/metrics/cyclomatic/elixir.rs @@ -164,8 +164,9 @@ impl Cyclomatic for ElixirCode { // A `receive` is a `case` over the mailbox and // pays as one (#1581), when it has message // clauses to pick between: one holding only an - // `after` timeout is a timed wait. The timeout - // clause is the default arm and free in standard + // `after` timeout is a timed wait. Standard pays + // the timeout clause only when a bare `_ ->` + // already took the one free default arm // (`elixir_clause_is_decision`). "receive" if elixir_receive_dispatches(node, code) => { stats.cyclomatic_modified += 1.; diff --git a/src/metrics/elixir_decisions.rs b/src/metrics/elixir_decisions.rs index 2c58197c..2b895302 100644 --- a/src/metrics/elixir_decisions.rs +++ b/src/metrics/elixir_decisions.rs @@ -333,14 +333,21 @@ fn elixir_is_first_clause<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> /// `fn`'s first, except the construct's free default clause /// ([`elixir_is_default_clause`]). /// -/// A `receive`'s `after` clause is free too, in every spelling: it is -/// the timeout, the path taken when no message clause matched in time, -/// which makes it `receive`'s default arm — the role a bare `_ ->` plays -/// in a `case` (#1581). It is never a dispatch of its own: the compiler -/// accepts exactly one `after` clause, and its head is a timeout value, -/// not a pattern that can fail. No other construct puts a clause there, -/// since `try`'s `after` takes a plain body, so the section is simply -/// not among those whose clauses count. +/// A `receive` gets exactly one free arm, in every spelling. Its +/// `after` clause is the timeout, the path taken when no message clause +/// matched in time, which makes it `receive`'s default arm — the role a +/// bare `_ ->` plays in a `case` (#1581). But a bare `_ ->` in the `do` +/// section is free under `receive` as well, and with one present the +/// timeout is a path of its own: `receive do {:a, y} -> y; _ -> 0 after +/// 10 -> :t end` reaches a matched message, any other message, and an +/// empty mailbox timing out — three paths, the three-clause `case`'s +/// two decisions. So the `after` clause is free only when the `do` +/// section has no bare catch-all ([`elixir_receive_after_is_decision`]). +/// It is never a dispatch in its own right: the compiler accepts +/// exactly one `after` clause, and its head is a timeout value, not a +/// pattern that can fail. No other construct puts a clause there, since +/// `try`'s `after` takes a plain body, so the section is not among +/// those [`DECISION_SECTION_KEYS`] names. /// /// An anonymous fn's head clause is the closure's definition, not a /// dispatch: the closure opens its own function space whose base path @@ -382,6 +389,9 @@ pub(super) fn elixir_clause_is_decision<'a>( ) -> bool { use Elixir as E; + if let Some(receive) = elixir_after_section_receive(node, code, ancestors) { + return elixir_receive_after_is_decision(&receive, code); + } let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); let in_section = up.next().is_some_and(|parent| { matches!( @@ -422,11 +432,55 @@ pub(super) fn elixir_clause_is_decision<'a>( /// clause by clause with the test below. pub(super) fn elixir_with_else_dispatches(call: &Node, code: &[u8]) -> bool { elixir_section_has_clause(call, code, Section::Else, |clause| { - !elixir_sole_unguarded_pattern(clause) - .is_some_and(|pattern| elixir_is_bare_wildcard(&pattern, code)) + !elixir_is_catch_all(clause, code) + }) +} + +/// Whether `clause` (a `stab_clause`) is a bare, unguarded `_ ->`. +fn elixir_is_catch_all(clause: &Node, code: &[u8]) -> bool { + elixir_sole_unguarded_pattern(clause) + .is_some_and(|pattern| elixir_is_bare_wildcard(&pattern, code)) +} + +/// Whether a `receive`'s `after` clause is a decision: whether its `do` +/// section already holds the free bare `_ ->`, leaving the timeout as a +/// path of its own. Without one the timeout is the construct's single +/// free arm ([`elixir_clause_is_decision`]). +fn elixir_receive_after_is_decision(receive: &Node, code: &[u8]) -> bool { + elixir_section_has_clause(receive, code, Section::Do, |clause| { + elixir_is_catch_all(clause, code) }) } +/// The `receive` Call whose `after` section holds `node` (a +/// `stab_clause`): the block form's `after_block` inside the Call's +/// `do_block`, or the keyword form's `after: (…)` block, bare or +/// bracketed. `None` for a clause anywhere else, and for an `after` +/// section of any other Call. +fn elixir_after_section_receive<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> Option> { + use Elixir as E; + + let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); + let call = match up.next().map(|parent| parent.kind_id().into()) { + Some(E::AfterBlock) => up + .next() + .filter(|do_block| do_block.kind_id() == E::DoBlock as u16) + .and_then(|_| up.next()), + Some(E::Block) => { + let pair = up.next()?; + elixir_pair_section(&pair, code, up) + .filter(|(key, _)| *key == "after") + .map(|(_, call)| call) + } + _ => None, + }?; + (elixir_call_keyword(&call, code) == Some("receive")).then_some(call) +} + /// Whether a `receive` Call dispatches over the mailbox: whether its /// `do` section holds a message clause, in any spelling. One holding /// only an `after` timeout (`receive do after 100 -> :ok end`, the diff --git a/src/metrics/elixir_spelling_parity_tests.rs b/src/metrics/elixir_spelling_parity_tests.rs index 6bb5aa26..2d8a08d8 100644 --- a/src/metrics/elixir_spelling_parity_tests.rs +++ b/src/metrics/elixir_spelling_parity_tests.rs @@ -390,7 +390,8 @@ fn elixir_for_reduce_head_is_free_in_every_spelling() { /// timeout counted as one more clause. The timeout is the path taken /// when no clause matched in time — `receive`'s default arm, which /// `case`'s `_ ->` twin shows is free — so with it the `receive` still -/// scores what the two-clause `case` does. +/// scores what the two-clause `case` does. A `receive` has one free arm, +/// though, so beside a bare `_ ->` the timeout counts. #[cfg(feature = "elixir")] #[test] fn elixir_receive_scores_like_case() { @@ -413,6 +414,22 @@ fn elixir_receive_scores_like_case() { "receive(do: ({:a, y} -> if(y, do: 1)), after: (10 -> :t))", ], ); + // A `receive` gets one free arm. With a bare `_ ->` in its `do` + // section the timeout is a path of its own: a matched `{:a, y}`, any + // other message, and an empty mailbox timing out make three paths, + // so cyclomatic is 3 (two decisions: `{:a, y}` and `after`, the `_` + // free), modified 2 (base + the container), C 2 (one per decision), + // cognitive 1 (+nesting on the container, arms free) — exactly the + // three-clause `case` beside it, whose `_ ->` is its free arm. + assert_group( + "case x do {:a, y} -> y; :b -> 0; _ -> 1 end", + [3, 2, 2, 1], + &[ + "receive do {:a, y} -> y; _ -> 0 after 10 -> :t end", + "receive(do: ({:a, y} -> y; _ -> 0), after: (10 -> :t))", + "receive([do: ({:a, y} -> y; _ -> 0), after: (10 -> :t)])", + ], + ); // A lone catch-all is free in standard and still pays the container. assert_group( "case x do _ -> 0 end", diff --git a/src/metrics/nexits.rs b/src/metrics/nexits.rs index 8716a4a5..c2916cf4 100644 --- a/src/metrics/nexits.rs +++ b/src/metrics/nexits.rs @@ -1760,15 +1760,14 @@ mod tests { #[cfg(feature = "perl")] #[test] fn perl_exit_key_in_a_flat_error_scores_like_its_twin() { - // Past `WIDE_PARENT` in `big-code-analysis-ast/src/node/ancestors.rs`, - // the width from which a walk records a node's children. - const RECORDED_WIDTH: usize = 64; + // The width from which a walk records a node's children. + use big_code_analysis_ast::node::WIDE_PARENT; let key = format!("my @r = ({}", "{\ndie => 1,\nexit,\n".repeat(12)); let twin = key.replace("die =>", "'die' =>"); let ast = Ast::parse(Source::new(LANG::Perl, key.as_bytes())).expect("perl is enabled"); let root = ast.as_tree_sitter().root_node(); assert!( - root.is_error() && root.child_count() > RECORDED_WIDTH, + root.is_error() && root.child_count() > WIDE_PARENT, "the fixture must parse to a wide flat ERROR root" ); assert_eq!(perl_nexits(&twin), 12); diff --git a/src/metrics/perl_keyword_key_tests.rs b/src/metrics/perl_keyword_key_tests.rs index c5cb9577..0b9b37b4 100644 --- a/src/metrics/perl_keyword_key_tests.rs +++ b/src/metrics/perl_keyword_key_tests.rs @@ -902,15 +902,14 @@ fn perl_bareword_outside_a_key_keeps_its_call() { #[cfg(feature = "perl")] #[test] fn perl_flat_error_reads_siblings_from_the_walk() { - // Past `WIDE_PARENT` in `big-code-analysis-ast/src/node/ancestors.rs`, - // the width from which a walk records a node's children. - const RECORDED_WIDTH: usize = 64; + // The width from which a walk records a node's children. + use big_code_analysis_ast::node::WIDE_PARENT; let body = "{\nlast,\n$a && $b,\n- and,\n-a => 1,\nname => 1,\n$c > $d && $e,\n"; let source = format!("my @r = ({}", body.repeat(12)); let ast = Ast::parse(Source::new(LANG::Perl, source.as_bytes())).expect("perl is enabled"); let root = ast.as_tree_sitter().root_node(); assert!( - root.is_error() && root.child_count() > RECORDED_WIDTH, + root.is_error() && root.child_count() > WIDE_PARENT, "the fixture must parse to a wide flat ERROR root" ); From 8df52071efbb43d9af59969a9fe9a98c7c73ea24 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:12:35 -0700 Subject: [PATCH 25/34] docs(lessons): record the vocabulary batch's traps Add lessons 96 (a tree-sitter child seek is O(log n) only through a grammar repetition, #1584) and 97 (a before/after on a rebuilt corpus measures the corpus, #1569), and extend 53 (fixed-depth ancestor climb, #1574), 57 (reserved word folded into the name kind, #1568), 64 (two candidate default arms, #1581), 82 (reuse the caller's chain, #1580) and 95 (a real token wrapped in an extra ERROR, #1572). --- docs/development/lessons_learned.md | 144 +++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 5 deletions(-) diff --git a/docs/development/lessons_learned.md b/docs/development/lessons_learned.md index f8a3e1e1..dd42a995 100644 --- a/docs/development/lessons_learned.md +++ b/docs/development/lessons_learned.md @@ -139,6 +139,8 @@ number and the higher number stays as a redirect. | [93](#93-a-gate-that-reads-a-typed-accessor-is-invisible-to-the-front-ends-that-read-the-wire) | A gate that reads a typed accessor is invisible to the front-ends that read the wire | | [94](#94-a-sweep-that-finds-nothing-only-rules-out-what-its-predicate-can-express) | A sweep that finds nothing only rules out what its predicate can express | | [95](#95-error-recovery-inserts-tokens-that-is_missing-does-not-flag) | Skip invented tokens by width, not `is_missing()` — valid code triggers it too | +| [96](#96-a-tree-sitter-child-seek-is-olog-n-only-through-a-grammar-repetition) | A tree-sitter child seek is O(log n) only through a grammar repetition — the walk is the O(1) source of siblings | +| [97](#97-a-beforeafter-on-a-rebuilt-corpus-measures-the-corpus) | A before/after on a rebuilt corpus measures the corpus — run the old binary on it too | --- @@ -2112,7 +2114,11 @@ slot from positional to field addressing, dump the AST for *each* spelling of the construct before choosing the fixture: two spellings that look interchangeable can put the `;` at different indices, and an empty-slot regression test written against the wrong one passes -vacuously under the old code. +vacuously under the old code. The same rule holds upward: a fixed-depth +parent climb (`up.nth(2)`) is positional addressing of an ancestor. +Climb by role, stepping over an optional wrapper, and verify the +identity top-down — that the node reached is the one the construct's +own accessor returns. Each language's grammar makes a different choice about which slots are optional, so the bug is per-language and per-statement-kind, not @@ -2148,6 +2154,17 @@ deeper, every `*_inspect_container` but Tcl's and iRules' still unwraps with `node.child(1)`, so `if ((/* n */ a))` scores zero there — #1181's defect inside the wrapper, left open. +**Elixir's section lookup climbed exactly two parents** (#1574). A +trailing bracketed keyword list `[do: …, else: …]` inserts a `list` +between `keywords` and `arguments`, so the climb from a pair landed on +`arguments` instead of the Call and answered "no owner" plausibly: +`with` lost its modified-cyclomatic `else` tier, `for … reduce:` scored +its accumulator head as a decision, `cond` lost its free `true ->`, and +keyword-form `if` / `unless` lost ABC's `else` condition. The fix climbs +by role over the optional `list` and accepts the node only when +`elixir_trailing_keywords` returns the same pair from the top — which is +also what rejects a non-trailing or nested list. + --- ## 54. A "no-op regen" must be proven by an actual regen + diff @@ -2258,7 +2275,10 @@ access"). A shape-only proxy passes every test whose fixtures happen to use the expected identity and silently mis-handles every other receiver; if you must approximate, document it as an under-approximation and prove the boundary with a fixture using a *foreign* receiver. Do not reach for -`to_string_lossy()` here (lesson 43). +`to_string_lossy()` here (lesson 43). When a grammar folds a reserved +word into the name kind, gate by text in every position the word can +occupy and exclude only the positions the language specification allows +it as a name — not the one kind the issue happens to name. "An `Attribute` whose first child is an `Identifier`" *looks* like `self.x`, but it is equally `db.x` and `logger.x`. The shape answers "is @@ -2277,6 +2297,17 @@ inflated NPA directly. The fix matches receiver bytes against the dedup key, and separates `self.f.g = 1` (an attribute on `self.f`) from `self.g = 1` — a distinction the structural proxy could never make. +**Groovy billed its primitive types as operands** (#1568). The grammar +has no primitive node: `int` is a `type_identifier` in a declaration but +a plain `identifier` in `int.class` and in an uninitialised `int t`, so +the issue's proposed `type_identifier`-only gate would have left half +the shapes diverging from Java. The fix classifies the nine spellings by +text wherever they appear and excludes the name positions the Groovy +spec permits — after a member-access operator (`x.int`, `x?.int`, +`x*.int`, `x::int`) and as a map or named-argument label (`[int: 1]`, +`f(int: 1)`) — pinned by a Java-twin test on shared declarations and by +the class-literal rows in one shared expectation table. + --- ## 58. A wrapper-node + keyword leaf is one operator, not two @@ -2538,7 +2569,10 @@ increment — so its default really is redundant — before suppressing, and confirm the shared node kind is not also serving a chain construct where the arm is a legitimate +1. See [`grammar-dispatch.md` §8](../../.claude/rules/grammar-dispatch.md) and -lesson 11 for the cross-metric agreement rule. +lesson 11 for the cross-metric agreement rule. A construct can offer two +candidate defaults; exactly one is free. Decide by counting reachable +paths, and write the fixture that carries both arms — one that carries +either is satisfied by the wrong rule. **Ruby `case`-else is +0 but `begin`/`rescue`-else is +1** (#451, `30a435ae`). The shared `R::Else` node appears under three constructs. A @@ -2551,6 +2585,16 @@ plan proposed suppressing rescue-else too**; doing so would have introduced a *fresh* divergence from Python. The fix gates only on `parent ∈ {Case, CaseMatch}`. +**Elixir `receive` with both `_ ->` and `after` had two free arms** +(#1581, caught by the whole-branch review). #1581 made the `after` +timeout clause free as "receive's default arm, the role `_ ->` plays in +a `case`" — but `_ ->` stayed free under `receive` too. `receive do +{:a, y} -> y; _ -> 0 after 10 -> :t end` has three reachable paths and +scored one decision, the same as a two-path `case`. The per-issue +review and `audit-tests` both passed, because every fixture had one arm +or the other and never both. The rule is now one free arm: the catch-all +if the `do` section has one, otherwise `after`. + --- ## 65. Removing a node kind from `is_func` / `is_func_space` zeroes its childless variant @@ -3188,7 +3232,12 @@ wrong answers rather than a failure, and a parity test written against a step `O(1)`, not the climb: a predicate that walks "until the first ancestor that is not X" is still `O(depth)` per node wherever X can nest without bound, so propagate the *answer* instead, and add a -`bench-scaling` probe of that shape before trusting the gate. +`bench-scaling` probe of that shape before trusting the gate. And +before deriving a chain for a predicate that lacks one, check whether +its caller's walk already carries it: the `find` / `count` `call` +filter had `ancestors` in hand all along, so a `_with_code` spelling +that forwards by default was cheaper, and linear, where a new +`Node::parent` climb would have been neither (#1580). `tree_sitter` stores no parent pointer — `Node::parent` restarts at the root and descends — so any predicate asking a node for an ancestor is @@ -3898,7 +3947,12 @@ what the author wrote must key on **width** (`start_byte() == end_byte()`), not on `Node::is_missing()`. Prove it with one malformed fixture per grammar the guard covers. Do not assume the trap needs invalid input: grammars cannot parse every valid program, and they -recover from the gap the same way. +recover from the gap the same way. The inverse trap is a *real* token +that recovery wraps in an `ERROR` it marks as an extra: a sibling helper +written to step over comments steps over the token too. Skip +`is_extra() && !is_error()` only, and prove it with a fixture whose +recovery puts an `ERROR` between the subject and the token the helper +looks for. When tree-sitter recovers from a parse error, it can insert a token the source lacks, such as a `;`, a `)` or an identifier. The inserted token @@ -3926,4 +3980,84 @@ contain one. valid trees contain zero-width nodes too, such as an empty Bash heredoc body. +**A stray `,` before `=>` hid a real token inside an extra** (#1572). +`(name, => 1)` is valid Perl — consecutive commas are legal, and `=>` +quotes only the word immediately to its left, so `name` is a call — but +the grammar recovers by wrapping the `,` in an `ERROR` flagged +`is_extra()`. The key-position predicate's `next_non_extra`, written to +look past comments, skipped it and quoted `name` as a hash key. The +fix's review caught it; the helper now passes over comments only, as +the rule `perl_not_is_key` beside it already did. + +--- + +## 96. A tree-sitter child seek is O(log n) only through a grammar repetition + +**Lesson:** Answer "which sibling sits next to this child" from the walk +that pushed the children, never from a tree-sitter seek on the parent. +`goto_first_child_for_byte`, `Node::child(i)`, `next_sibling` and +`goto_previous_sibling` are O(log n) only when the parent's children are +stored as a grammar `repeat()`'s balanced hidden subtree. Over a flat +child array — every `ERROR` that recovery builds by skipping tokens — +each is O(index), and a per-child sibling question is quadratic. Record a +wide node's children when the walk pushes them, lend the record through +the ancestor chain, and gate the shape with a `bench-scaling` probe whose +body really flattens: plain repetition folds into nested +`binary_expression`s and keeps its repeat structure, so a probe needs a +stray `{` or `$d` per unit before it measures the flat case. + +The cost is invisible on the inputs anyone tests. Valid code and an +unclosed list both keep the repetition's hidden levels, so a seek that +is documented O(log n) is O(log n) there, and a timing probe written +from the same shapes fits linear. Only an `ERROR` holding its children +as one flat array exposes the scan — and real files produce one: a +Perl debugger script whose tail fails to parse has 30,798 direct +children under its root. + +**Perl ABC and Halstead were quadratic under a flat `ERROR`** (#1584, +found while writing #1575's probes). `children_after` / +`children_before` from #1545 and #1558 carried an O(log n) claim that +held for the unclosed-list probe written beside them, because that list +kept its repeat structure. With a `{` per body the root became a flat +`ERROR` and the bareword-key and misparse lookups ran 4.07× per +doubling, 8.99 s at 32,000 keys. Halstead and `ops` hit the same scan +through the dash-key and `not`-key helpers, and +`Ancestors::previous_sibling` made Kotlin Halstead quadratic on a +*valid* string of 8,000 `$a` interpolations (11.47 s). The fix records +each node with more than `WIDE_PARENT` children into a per-depth +`SiblingCache` as the metric, `ops` and `find`/`count` walks push them, +and every lookup falls back to the old seek on a miss; five probes trip +at 1.7–2.1 with the cache ignored and fit 0.9–1.1 with it. + +--- + +## 97. A before/after on a rebuilt corpus measures the corpus + +**Lesson:** When re-deriving a calibrated default after a metric +definition changes, measure the rebuilt corpus with the previous release +binary as well as the new one, and attribute each row's movement to the +metric or to the sample before touching a limit or a published table. +Record the corpus — repositories, commit SHAs, exclusion globs — in the +repository, so the next re-derivation has an identical sample to run +both binaries against instead of a comparable one. + +A percentile of a sampled corpus moves with the sample. A metric change +and a sample change present identically — the per-language rows moved — +and the issue that asks for the re-derivation presupposes the first +cause, because the metric change is what prompted it. Without the +old-binary run on the same files there is nothing to separate the two, +and the stale-looking rows read as drift to correct. + +**The `halstead.effort` default survived a re-derivation that looked +like a 5× drift** (#1569). #1140's 43-repository corpus had never been +recorded, so a comparable 41-repository one was rebuilt. Against the +book's rows it moved 2–5× (C# 4.6×), and the issue's own evidence called +Java's rationale "off by about 5×". Run with the v2.3.0 binary on the +identical rebuilt files, the vocabulary changes moved p97.5 by at most +47% (Python) and the median-language flag rate from 2.23% to 2.20%: +almost all of the movement was the change of repositories. The default +stayed at 50,000, the book's column was regenerated from the new +corpus, and the corpus is recorded in +`docs/development/threshold_corpus.md`. + --- From bda3dede4907dabfad12c90621a6d675e25f66c5 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:57:22 -0700 Subject: [PATCH 26/34] fix(halstead): apply the max code-review findings Bill the vendor spelling `__volatile__` (Volatile2) in the C, C++, Mozcpp and ObjC modifier arms, so `__asm__ __volatile__` scores like `asm volatile`. Read an Elixir keyword pair's section off the ancestor chain instead of rescanning `arguments` per pair or clause, which was O(P*K) on wide calls (5.4 s -> 0.06 s at 16k arguments), bound the bracketing-list climb to one step, and share one section-holder helper across cyclomatic, ABC and cognitive. Pin that Perl's is_call_with_code disagrees with the byte-less is_call on a key or wrapper callee, pin the `(-foo, => 1)` and comment-in-slice shapes, and record the ast-crate API changes and the cognitive scope of #1574 in the changelog. Add the cognitive/elixir-mixed-arguments probe. --- CHANGELOG.md | 41 ++- big-code-analysis-ast/src/checker.rs | 70 ++++- big-code-analysis-ast/src/getter/c.rs | 8 +- big-code-analysis-ast/src/getter/cpp.rs | 2 +- big-code-analysis-ast/src/getter/csharp.rs | 30 ++- big-code-analysis-ast/src/getter/kotlin.rs | 15 +- big-code-analysis-ast/src/getter/mozcpp.rs | 2 +- big-code-analysis-ast/src/getter/objc.rs | 14 +- big-code-analysis-ast/src/getter/perl.rs | 6 +- .../src/lang_helpers/elixir.rs | 78 +++--- .../src/lang_helpers/perl.rs | 62 +++-- big-code-analysis-bench/src/shapes.rs | 88 ++++++ big-code-analysis-book/src/metrics.md | 19 +- docs/development/benchmarking.md | 3 +- docs/development/threshold_corpus.md | 3 +- src/metrics/abc/elixir.rs | 33 +-- src/metrics/cognitive/elixir.rs | 14 +- src/metrics/cyclomatic/elixir.rs | 5 +- src/metrics/elixir_decisions.rs | 255 ++++++++++-------- src/metrics/elixir_spelling_parity_tests.rs | 31 +-- src/metrics/halstead.rs | 7 + src/metrics/nexits.rs | 16 +- src/metrics/perl_keyword_key_tests.rs | 37 ++- src/ops.rs | 7 + 24 files changed, 565 insertions(+), 281 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fb05692..fd4f6ddc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ for historical reference. ### Added -- **Twelve Perl, Bash and Kotlin scaling probes in the +- **Thirteen Perl, Bash, Kotlin and Elixir scaling probes in the complexity-class gate** (`make bench-scaling`). #1575 adds seven: the Perl ABC misparse climb, Perl ABC's `=~` binding check under an `ERROR` root, and the Bash cognitive / ABC comparison-run walks @@ -38,7 +38,11 @@ for historical reference. and `halstead/kotlin-short-interpolations` (#1584), `count/perl-error-tail-calls` (#1580) and `nexits/perl-error-tail-keys` (#1585). Each was watched tripping with - its pre-fix walk restored. + its pre-fix walk restored. `cognitive/elixir-mixed-arguments` prices + the keyword-section lookup #1574 added, whose first form read a call's + whole argument list once per keyword pair and once per keyword-form + clause: 5.4 s for a 16 000-argument call in a release build, now + linear. ### Changed @@ -111,9 +115,10 @@ for historical reference. `register`, `inline`, `thread_local`, `const`, `volatile`, `restrict`, `_Atomic`, `constexpr`, `_Noreturn`, `alignas`, their vendor spellings (`__inline`, `__forceinline`, `__restrict__`, `__thread`, - `__extension__`, …) and C's storage class `auto`. C++ and Mozcpp also - bill `virtual`, `mutable`, `constinit`, `consteval`, `explicit`, - `friend`, the access labels, `override`, `final` and the `noexcept` + `__volatile__`, `__extension__`, …) and C's storage class `auto`. C++ + and Mozcpp also bill `virtual`, `mutable`, `constinit`, `consteval`, + `explicit`, `friend`, the access labels, `override`, `final` and the + `noexcept` of an exception specification. Each counts once as its keyword, never as the `storage_class_specifier` / `type_qualifier` / `virtual_specifier` / `access_specifier` wrapper. `override` and @@ -143,9 +148,12 @@ for historical reference. ends a longer expression before `=>` (`(1 - name => 2)`), since `=>` quotes the word right before it. Package-qualified words (`Foo::Bar => 1`), words with arguments (`name() => 1`), bareword - values (`(a => foo)`) and `$h{+shift}` keep their call. Cyclomatic and - cognitive are unchanged. This is a local workaround for upstream - #1571. + values (`(a => foo)`) and `$h{+shift}` keep their call. So does a + `-name` outside those two positions — a value (`(x => -name)`), a + comma-separated argument (`f(-text, 'hi')`) or a multi-key slice — + since there Perl's own reading depends on whether a sub of that name + is visible, which no single file says. Cyclomatic and cognitive are + unchanged. This is a local workaround for upstream #1571. - **Perl `bca find --type call` and `bca count --type call` report each call once and skip auto-quoted keys** (#1580). tree-sitter-perl parses a call's name (`foo` in `foo(1)`, `print` in `print "x"`) as a bareword @@ -177,10 +185,16 @@ for historical reference. and a user module's `Foo.||(a, b)` still scores nothing. Cognitive now scores the keyword forms' `else:` (+1) and `rescue:` / `catch:` (+nesting) for `if`, `unless`, `try` and the `def` family, as their - blocks do; the keyword `try` scored 0 before. A `with`'s `else` adds + blocks do; the keyword `try` scored 0 before. For any other owner the + keyword form still scores nothing — `else:` is an ordinary option to + a function — while its `do … end` block scores as before, so + `foo(do: 1, else: 2)` and `foo do 1 else 2 end` still differ by one + there. A `with`'s `else` adds no cognitive cost in either spelling: its clauses are the arms of a `case`, already paid by the container (the block form scored 2 - against the keyword form's 1). + against the keyword form's 1). In `big-code-analysis-ast`, + `elixir_remote_call_applies` is replaced by + `elixir_applying_remote_call`, which returns the applying call. - **Elixir `receive` scores like `case`** (#1581): modified cyclomatic counts the construct once and cognitive complexity adds +nesting, in the block and keyword forms. A `receive` gets one free arm, as a @@ -201,7 +215,12 @@ for historical reference. of seeking from the `ERROR`'s start: 16 000 copies of `{ last,` take 0.03 s, down from 2.2 s. The same change makes Kotlin Halstead linear on a string of many short `$name` interpolations, which was quadratic - on valid input (8 000 names: 11.5 s → 0.01 s). + on valid input (8 000 names: 11.5 s → 0.01 s). In + `big-code-analysis-ast`, which `STABILITY.md` leaves outside the shape + contract, `AncestorIter::Chain` carries the lent cache as a second + field, `perl_dash_signs_keyword_key` takes the `-`'s ancestor chain in + place of its parent node, and `perl_is_dash_key_word` is + crate-private. ## [2.3.1] - 2026-10-10 diff --git a/big-code-analysis-ast/src/checker.rs b/big-code-analysis-ast/src/checker.rs index 1b747b01..c22c00f5 100644 --- a/big-code-analysis-ast/src/checker.rs +++ b/big-code-analysis-ast/src/checker.rs @@ -513,7 +513,11 @@ pub trait Checker { /// Perl overrides it: the grammar nests a call's name inside the /// call as a bareword call of its own, and parses an auto-quoted /// hash key as one, so only the position says which words are - /// calls (#1580). + /// calls (#1580). As with [`is_string_with_code`](Self::is_string_with_code), + /// a call site reaching for the byte-less spelling reads as correct + /// against every language without an override; + /// `perl_bareword_is_a_call_only_to_the_byteless_spelling` pins that + /// the two really do disagree. #[inline] #[must_use] fn is_call_with_code<'a>(node: &Node<'a>, _code: &[u8], _ancestors: Ancestors<'a, '_>) -> bool { @@ -3542,6 +3546,70 @@ mod tests { ); } + /// A Perl bareword is a call to [`Checker::is_call`] and, as a + /// wrapper's callee or an auto-quoted key, not to + /// [`Checker::is_call_with_code`] (#1580). The end-to-end + /// `find --type call` tests observe the result, which is equally + /// consistent with the override never firing; this asserts the + /// disagreement, so a call site reaching for the byte-less spelling + /// is wrong rather than indistinguishable (grammar-dispatch §7). + /// + /// Three structurally independent routes withdraw a bareword — key + /// position, the spaced wrapper's callee, the bracketed wrapper's + /// callee — and two keep one: a bareword argument and a lone + /// bareword (grammar-dispatch §11). The two wrappers are the control + /// for the kind gate: both spellings must still call them. + #[test] + #[cfg(feature = "perl")] + fn perl_bareword_is_a_call_only_to_the_byteless_spelling() { + let code = b"my %h = (name => 1);\nprint shift;\nfoo(1);\nfoo;\n"; + let (mut kept, mut withdrawn, mut other) = (Vec::new(), Vec::new(), 0); + for_each_node_with_chain::(code, |node, chain| { + let known = PerlCode::is_call_with_code(node, code, Ancestors::known(chain)); + assert_eq!( + known, + PerlCode::is_call_with_code(node, code, Ancestors::unknown()), + "known chain and parent climb disagree on {} at row {}", + node.kind(), + node.start_row() + ); + if node.kind_id() != Perl::CallExpressionWithBareword as u16 { + assert_eq!( + known, + PerlCode::is_call(node), + "the override moved a {} node at row {}", + node.kind(), + node.start_row() + ); + other += usize::from(known); + return; + } + assert!( + PerlCode::is_call(node), + "the kind table must still list the bareword kind, or this test is \ + measuring its absence rather than the override" + ); + let text = + String::from_utf8_lossy(&code[node.start_byte()..node.end_byte()]).into_owned(); + if known { + kept.push(text); + } else { + withdrawn.push(text); + } + }); + assert_eq!( + kept, + ["shift", "foo"], + "a bareword argument and a lone bareword are calls" + ); + assert_eq!( + withdrawn, + ["name", "print", "foo"], + "an auto-quoted key and the callee inside either wrapper are not" + ); + assert_eq!(other, 2, "the two wrappers stay calls to both spellings"); + } + /// The value slots of the three constructs whose argument lists hold /// both roles stay string literals, though `is_value_braced_word` /// classifies each construct whole as script-taking diff --git a/big-code-analysis-ast/src/getter/c.rs b/big-code-analysis-ast/src/getter/c.rs index eaf3876f..9131eb3b 100644 --- a/big-code-analysis-ast/src/getter/c.rs +++ b/big-code-analysis-ast/src/getter/c.rs @@ -93,8 +93,10 @@ impl Getter for CCode { // `storage_class_specifier` / `type_qualifier` / // `alignas_qualifier`, so it bills once, as `const` does in // a pointer declarator (`int * const p`). The vendor - // spellings (`__inline`, `__restrict__`, `__thread`, …) are - // kinds of their own and bill under their own name. `auto` + // spellings (`__inline`, `__restrict__`, `__thread`, + // `__volatile__`, …) are kinds of their own and bill under + // their own name; `__volatile__` the grammar emits only as a + // GNU `asm` qualifier, where `volatile` bills too. `auto` // is a storage class in C, the only place this grammar // emits it. `__extension__` bills in an expression too, // where it is the prefix of an `extension_expression`. @@ -109,7 +111,7 @@ impl Getter for CCode { // operand does — the grammar decides that, not this arm, as // `objc.rs` notes for `nullable`. | Static | Extern | Register | Auto | Inline | Inline2 | Inline3 | Forceinline - | ThreadLocal | Thread | Const | Constexpr | Volatile | Restrict | Restrict2 + | ThreadLocal | Thread | Const | Constexpr | Volatile | Volatile2 | Restrict | Restrict2 | Atomic | Noreturn | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension => TokenRole::Operator, // `CharLiteral` joins the operand list here for the whole diff --git a/big-code-analysis-ast/src/getter/cpp.rs b/big-code-analysis-ast/src/getter/cpp.rs index 7c17ee15..60821e3c 100644 --- a/big-code-analysis-ast/src/getter/cpp.rs +++ b/big-code-analysis-ast/src/getter/cpp.rs @@ -227,7 +227,7 @@ impl Getter for CppCode { // the grammar emits it only under // `placeholder_type_specifier`. | Static | Extern | Register | Inline | Inline2 | Inline3 | Forceinline - | ThreadLocal | Thread | Const | Constexpr | Volatile | Restrict | Restrict2 + | ThreadLocal | Thread | Const | Constexpr | Volatile | Volatile2 | Restrict | Restrict2 | Atomic | Noreturn | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension | Virtual | Mutable | Constinit | Consteval | Explicit | Friend | Public | Private | Protected | Override | Final | Noexcept2 diff --git a/big-code-analysis-ast/src/getter/csharp.rs b/big-code-analysis-ast/src/getter/csharp.rs index adf1b8b6..cfb5aa0d 100644 --- a/big-code-analysis-ast/src/getter/csharp.rs +++ b/big-code-analysis-ast/src/getter/csharp.rs @@ -25,18 +25,30 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { // a call or indexer argument without `out`. `F(out List x)`, // `(List p, int q) = …` and `o is List x` keep their pair. A // member access before the `<` (`F(x.a < b, c > d)`) wraps the generic -// name in `qualified_name`s, which the climb passes. The `>=` spelling +// name in a `qualified_name`, whose final name alone can be the misparse. +// Generic qualifiers keep their type brackets. The `>=` spelling // (`F(a < b, c >= d)`) parses as an assignment instead and is not undone. fn csharp_comparison_misparse<'a>(list: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { let mut up = ancestors.iter(list).map(|(node, _)| node); - if up + let Some(name) = up .next() - .is_none_or(|name| name.kind_id() != Csharp::GenericName as u16) - { + .filter(|name| name.kind_id() == Csharp::GenericName as u16) + else { return false; - } - let mut up = up.skip_while(|node| node.kind_id() == Csharp::QualifiedName as u16); - let (Some(declaration), Some(argument), Some(arguments)) = (up.next(), up.next(), up.next()) + }; + let declaration = match up.next() { + Some(qualified) if qualified.kind_id() == Csharp::QualifiedName as u16 => { + if qualified + .child_by_field_name("name") + .is_none_or(|last| last.id() != name.id()) + { + return false; + } + up.next() + } + parent => parent, + }; + let (Some(declaration), Some(argument), Some(arguments)) = (declaration, up.next(), up.next()) else { return false; }; @@ -75,9 +87,7 @@ fn is_interpolation_clause_delimiter<'a>(node: &Node<'a>, ancestors: Ancestors<' Csharp::COMMA => Csharp::InterpolationAlignmentClause, _ => return false, }; - ancestors - .parent(node) - .is_some_and(|parent| parent.kind_id() == clause as u16) + ancestors.parent_has_kind(node, clause as u16) } impl Getter for CsharpCode { diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index b04fd312..1b879404 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -157,15 +157,12 @@ fn kotlin_modifier_was_written_as_one<'a>(node: &Node<'a>, ancestors: Ancestors< // `function_value_parameters`, so the sibling is found without a scan // from the list's start: that made a long parameter list quadratic. fn kotlin_next_token_is_inserted<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - let (mut current, mut current_ancestors) = (*node, ancestors); - ancestors - .iter(node) - .find_map(|(parent, above)| { - let next = current_ancestors - .siblings_after(¤t) - .find(|sibling| !sibling.as_tree_sitter().is_extra()); - (current, current_ancestors) = (parent, above); - next + std::iter::once((*node, ancestors)) + .chain(ancestors.iter(node)) + .find_map(|(level, level_ancestors)| { + level_ancestors + .siblings_after(&level) + .find(|sibling| !sibling.as_tree_sitter().is_extra()) }) .is_some_and(|mut next| { while let Some(first) = next.child(0) { diff --git a/big-code-analysis-ast/src/getter/mozcpp.rs b/big-code-analysis-ast/src/getter/mozcpp.rs index c16140e2..e701395a 100644 --- a/big-code-analysis-ast/src/getter/mozcpp.rs +++ b/big-code-analysis-ast/src/getter/mozcpp.rs @@ -140,7 +140,7 @@ impl Getter for MozcppCode { // twin (#1578). The Gecko `MOZ_*` annotation macros stay // unbilled, as attributes do. | Static | Extern | Register | Inline | Inline2 | Inline3 | Forceinline - | ThreadLocal | Thread | Const | Constexpr | Volatile | Restrict | Restrict2 + | ThreadLocal | Thread | Const | Constexpr | Volatile | Volatile2 | Restrict | Restrict2 | Atomic | Noreturn | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension | Virtual | Mutable | Constinit | Consteval | Explicit | Friend | Public | Private | Protected | Override | Final | Noexcept2 diff --git a/big-code-analysis-ast/src/getter/objc.rs b/big-code-analysis-ast/src/getter/objc.rs index 2d398f5b..248aa411 100644 --- a/big-code-analysis-ast/src/getter/objc.rs +++ b/big-code-analysis-ast/src/getter/objc.rs @@ -201,13 +201,13 @@ impl Getter for ObjcCode { | ATthrow | ATsynchronized | ATautoreleasepool | ATselector | ATencode | Struct | Union | Enum | Typedef | ATinterface | ATimplementation | ATprotocol | Static | Extern | Register | Auto | Inline | Inline2 | Inline3 | Forceinline | ThreadLocal - | Thread | Const | Constexpr | Volatile | Restrict | Restrict2 | Atomic | Noreturn - | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension | Const2 | Complex - | Complex2 | Nullable | Nullable2 | Nullable3 | NullableResult | NullUnspecified - | Nonnull2 | Autoreleasing | Block | Strong | UnsafeUnretained | Weak | Bridge - | BridgeRetained | BridgeTransfer | Kindof | Covariant | Contravariant | Out - | Inout | Bycopy | Byref | Oneway | ATpublic | ATprivate | ATprotected | ATpackage - | AToptional | ATrequired => TokenRole::Operator, + | Thread | Const | Constexpr | Volatile | Volatile2 | Restrict | Restrict2 | Atomic + | Noreturn | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension | Const2 + | Complex | Complex2 | Nullable | Nullable2 | Nullable3 | NullableResult + | NullUnspecified | Nonnull2 | Autoreleasing | Block | Strong | UnsafeUnretained + | Weak | Bridge | BridgeRetained | BridgeTransfer | Kindof | Covariant + | Contravariant | Out | Inout | Bycopy | Byref | Oneway | ATpublic | ATprivate + | ATprotected | ATpackage | AToptional | ATrequired => TokenRole::Operator, Identifier if is_property_attribute_name(node, ancestors) => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only diff --git a/big-code-analysis-ast/src/getter/perl.rs b/big-code-analysis-ast/src/getter/perl.rs index 9388c6c6..d4f0155e 100644 --- a/big-code-analysis-ast/src/getter/perl.rs +++ b/big-code-analysis-ast/src/getter/perl.rs @@ -261,8 +261,10 @@ impl Getter for PerlCode { /// /// Any other `-bareword` key (`(-name => 1)`, `$h{-name}`) bills its /// word spelled from the sign (#1572). That is a source slice, so a - /// sign spaced off its word (`(- name => 1)`, rare) keeps the space: - /// one operand, distinct from `name`, but not merged with `-name`. + /// sign spaced off its word (`(- name => 1)`, rare) keeps the space, + /// and a comment between the two (`(- # c⏎ name => 1)`) is carried + /// along in it: one operand, distinct from `name`, but not merged + /// with `-name`, and listed by `bca ops` as written. fn get_operand_id<'a>( node: &Node<'a>, code: &'a [u8], diff --git a/big-code-analysis-ast/src/lang_helpers/elixir.rs b/big-code-analysis-ast/src/lang_helpers/elixir.rs index b0667418..0f3e211e 100644 --- a/big-code-analysis-ast/src/lang_helpers/elixir.rs +++ b/big-code-analysis-ast/src/lang_helpers/elixir.rs @@ -114,14 +114,14 @@ pub fn elixir_applying_operator<'a>( .filter(|parent| parent.kind() == "binary_operator") } -/// The operator token a remote call on `Kernel` with arguments applies, -/// or `None` for any other node: the `||` of `Kernel.||(a, b)`, or of -/// `x |> Kernel.||(y)`, which the pipe rewrites to it. Such a call -/// expands the `Kernel` macro exactly as `a || b` does, so it -/// short-circuits and decides as the operator does, where a capture -/// (`&Kernel.||/2`, a call with no arguments) only names it. It has no -/// `binary_operator`, so its operands are the call's arguments, and under -/// a pipe the piped value, rather than `left` / `right` fields. +/// The name of the short-circuit operator a remote call on `Kernel` with +/// arguments applies, or `None` for any other node: the `||` of +/// `Kernel.||(a, b)`, or of `x |> Kernel.||(y)`, which the pipe rewrites +/// to it. Such a call expands the `Kernel` macro exactly as `a || b` +/// does, so it short-circuits and decides as the operator does, where a +/// capture (`&Kernel.||/2`, a call with no arguments) only names it. It +/// has no `binary_operator`, so its operands are the call's arguments, +/// and under a pipe the piped value, rather than `left` / `right` fields. /// /// The receiver must spell `Kernel` (or `Elixir.Kernel`): a user module /// may define `||/2` as an ordinary function, and `Foo.||(a, b)` neither @@ -131,15 +131,20 @@ pub fn elixir_applying_operator<'a>( /// `operator_identifier`, which is what is returned. `Kernel.and(a, b)` /// and `Kernel.or(a, b)` spell their names as a plain `identifier`, with /// no `And` / `Or` token to find, so that identifier is returned instead -/// (#1582); every other word name (`Kernel.not`, `Kernel.in`) is a -/// function or decides nothing, and is not found. Nor is the childless -/// `..`. +/// (#1582). Any other name — `Kernel.+`, `Kernel.!`, `Kernel.not`, +/// `Kernel.in` — is a function or decides nothing, and is not found; nor +/// is the childless `..`. [`elixir_short_circuit_name`] says which +/// operation the name applies. +/// +/// Every identifier a walk meets asks this of its call, so the receiver, +/// which turns nearly all of them away without a cursor, is read before +/// the call's children are scanned for its `arguments`. /// /// Matched by rule name: `dot` and `arguments` are aliased to several /// ids each (grammar-dispatch §1). #[must_use] -pub fn elixir_kernel_operator_token<'a>(call: &Node<'a>, code: &[u8]) -> Option> { - if call.kind() != "call" || !call.children().any(|child| child.kind() == "arguments") { +pub fn elixir_kernel_short_circuit_name<'a>(call: &Node<'a>, code: &[u8]) -> Option> { + if call.kind() != "call" { return None; } let dot = call.child_by_field_name("target").filter(|target| { @@ -152,11 +157,13 @@ pub fn elixir_kernel_operator_token<'a>(call: &Node<'a>, code: &[u8]) -> Option< }) })?; let name = dot.child_by_field_name("right")?; - if name.kind() == "operator_identifier" { - return name.child(0); - } - (name.kind_id() == Elixir::Identifier as u16 - && elixir_short_circuit_name(&name, code).is_some()) + let name = if name.kind() == "operator_identifier" { + name.child(0)? + } else { + name + }; + (elixir_short_circuit_name(&name, code).is_some() + && call.children().any(|child| child.kind() == "arguments")) .then_some(name) } @@ -175,7 +182,7 @@ pub fn elixir_short_circuit_kind(kind_id: u16) -> Option { } /// [`elixir_short_circuit_kind`] for an operator's name as -/// [`elixir_kernel_operator_token`] returns it: a token, or the +/// [`elixir_kernel_short_circuit_name`] returns it: a token, or the /// `identifier` spelling `Kernel.and` / `Kernel.or` (#1582), whose /// bytes decide. Ask it only of such a name — an identifier is an `and` /// operator nowhere else. @@ -191,29 +198,30 @@ pub fn elixir_short_circuit_name(name: &Node, code: &[u8]) -> Option { } } -/// The `Kernel` remote call that applies `op`, an operator's name — the -/// call [`elixir_kernel_operator_token`] answers `op` for — or `None` -/// when `op` sits anywhere else. A token's parents are the -/// `operator_identifier`, then the `dot`, then the call; the -/// `identifier` naming `Kernel.and` sits on the `dot` directly. Any -/// other node fails on the first one or two parents' kinds, so the -/// check is cheap enough to ask of every identifier a walk meets. +/// The `Kernel` remote call that applies `op`, a short-circuit operator's +/// name — the call [`elixir_kernel_short_circuit_name`] answers `op` for +/// — paired with that call's own ancestry, or `None` when `op` sits +/// anywhere else. A token's parents are the `operator_identifier`, then +/// the `dot`, then the call; the `identifier` naming `Kernel.and` sits +/// on the `dot` directly. Any other node fails on the first one or two +/// parents' kinds, so the check is cheap enough to ask of every +/// identifier a walk meets. #[must_use] -pub fn elixir_applying_remote_call<'a>( +pub fn elixir_applying_remote_call<'a, 'c>( op: &Node<'a>, code: &[u8], - ancestors: Ancestors<'a, '_>, -) -> Option> { - let mut up = ancestors.iter(op).map(|(ancestor, _)| ancestor); - let mut dot = up.next()?; + ancestors: Ancestors<'a, 'c>, +) -> Option<(Node<'a>, Ancestors<'a, 'c>)> { + let mut up = ancestors.iter(op); + let (mut dot, _) = up.next()?; if dot.kind() == "operator_identifier" { - dot = up.next()?; + dot = up.next()?.0; } if dot.kind() != "dot" { return None; } - let call = up.next()?; - elixir_kernel_operator_token(&call, code) + let (call, call_ancestors) = up.next()?; + elixir_kernel_short_circuit_name(&call, code) .is_some_and(|name| name.id() == op.id()) - .then_some(call) + .then_some((call, call_ancestors)) } diff --git a/big-code-analysis-ast/src/lang_helpers/perl.rs b/big-code-analysis-ast/src/lang_helpers/perl.rs index e40ffadf..1e4b4eec 100644 --- a/big-code-analysis-ast/src/lang_helpers/perl.rs +++ b/big-code-analysis-ast/src/lang_helpers/perl.rs @@ -131,6 +131,7 @@ pub(crate) fn perl_file_test_is_dash_key<'a>( operand.is_child(Perl::FatComma as u16) && operand.start_row() == test.start_row() } else { is_glued_word(&operand, test) + && is_plain_bareword(&operand) && ancestors .iter(test) .next() @@ -183,7 +184,7 @@ pub fn perl_bareword_is_key<'a>(word: &Node<'a>, ancestors: Ancestors<'a, '_>) - return false; } match ancestors.iter(word).next() { - Some((unary, above)) if is_dash_unary(&unary) => is_key_position(&unary, above), + Some((unary, above)) if dash_operator(&unary).is_some() => is_key_position(&unary, above), _ => is_key_position(word, ancestors), } } @@ -222,12 +223,12 @@ pub fn perl_bareword_is_call<'a>(word: &Node<'a>, ancestors: Ancestors<'a, '_>) /// the string, so it is no operator. #[must_use] pub(crate) fn perl_dash_signs_key<'a>(dash: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - ancestors.iter(dash).next().is_some_and(|(parent, above)| { - perl_dash_signs_keyword_key(dash, ancestors) - || (parent.kind_id() == Perl::UnaryExpression as u16 + perl_dash_signs_keyword_key(dash, ancestors) + || ancestors.iter(dash).next().is_some_and(|(parent, above)| { + parent.kind_id() == Perl::UnaryExpression as u16 && next_non_extra(dash, ancestors).is_some_and(|word| is_plain_bareword(&word)) - && is_key_position(&parent, above)) - }) + && is_key_position(&parent, above) + }) } /// The `-` that signs the `-bareword` key `identifier` spells, or @@ -245,9 +246,7 @@ pub(crate) fn perl_bareword_key_sign<'a>( .next() .filter(|(word, _)| word.kind_id() == Perl::CallExpressionWithBareword as u16)?; let (unary, _) = up.next()?; - let dash = unary - .child_by_field_name("operator") - .filter(|op| op.kind_id() == Perl::DASH as u16)?; + let dash = dash_operator(&unary)?; // `word` and `dash` are siblings, so they share an ancestry. perl_dash_signs_key(&dash, word_ancestors).then_some(dash) } @@ -298,10 +297,8 @@ pub fn perl_key_is_signed<'a>(key: &Node<'a>, ancestors: Ancestors<'a, '_>) -> b return perl_dash_signs_keyword_key(&dash, ancestors); } up.next().is_some_and(|(outer, _)| { - outer.child_by_field_name("operator").is_some_and(|dash| { - dash.kind_id() == Perl::DASH as u16 - && perl_dash_signs_keyword_key(&dash, parent_ancestors) - }) + dash_operator(&outer) + .is_some_and(|dash| perl_dash_signs_keyword_key(&dash, parent_ancestors)) }) }) } @@ -354,7 +351,11 @@ fn is_key_position<'a>(key: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { // As in `perl_not_is_key`, only comments are passed over: the // grammar wraps a stray `,` (`(name, => 1)`) in an `ERROR` it // marks as an extra, and that `,`, not the word, is then the - // token the `=>` quotes, which is nothing. + // token the `=>` quotes, which is nothing. A `-foo, =>` file-test + // key stops here too and reads as the grammar does (a file test + // on `oo`), as every `-bareword` outside key position does; Perl + // itself reads one as a string or a negated call by whether a sub + // of that name is visible, which no single file says. if let Some(next) = node_ancestors .siblings_after(&node) .find(|sibling| sibling.is_error() || !sibling.as_tree_sitter().is_extra()) @@ -371,21 +372,34 @@ fn is_key_position<'a>(key: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { // `identifier`, no package qualifier, no arguments. Only such a word is // auto-quoted. fn is_plain_bareword(word: &Node) -> bool { - word.kind_id() == Perl::CallExpressionWithBareword as u16 - && word + if word.kind_id() != Perl::CallExpressionWithBareword as u16 { + return false; + } + // Every call's name is one of these, so the common shape — one + // child, no comment inside the call — is read off two fields rather + // than through a cursor over the children. + match word.child_count() { + 1 => word + .child(0) + .is_some_and(|child| child.kind_id() == Perl::Identifier as u16), + _ => word .children() .filter(|child| !child.as_tree_sitter().is_extra()) .map(|child| child.kind_id()) - .eq([Perl::Identifier as u16]) + .eq([Perl::Identifier as u16]), + } } -// Whether `node` is a unary minus, the `unary_expression` a `-bareword` -// key whose letter is no file-test letter parses as. -fn is_dash_unary(node: &Node) -> bool { - node.kind_id() == Perl::UnaryExpression as u16 - && node - .child_by_field_name("operator") - .is_some_and(|op| op.kind_id() == Perl::DASH as u16) +// The `-` of `node` when it is a unary minus — the `unary_expression` a +// `-bareword` key whose letter is no file-test letter parses as — and +// `None` for any other node, a subtraction's `binary_expression` +// included. +fn dash_operator<'a>(node: &Node<'a>) -> Option> { + if node.kind_id() != Perl::UnaryExpression as u16 { + return None; + } + node.child_by_field_name("operator") + .filter(|op| op.kind_id() == Perl::DASH as u16) } fn non_extra_count(node: &Node) -> usize { diff --git a/big-code-analysis-bench/src/shapes.rs b/big-code-analysis-bench/src/shapes.rs index 6e846445..ece4a13d 100644 --- a/big-code-analysis-bench/src/shapes.rs +++ b/big-code-analysis-bench/src/shapes.rs @@ -34,6 +34,23 @@ use big_code_analysis::{Ast, CodeMetrics, LANG, Metric, MetricsError, MetricsOpt /// `base + per_unit * n`. See the module docs for why. pub type Render = fn(usize) -> String; +/// C#: a qualified type with one generic argument per segment. +#[must_use] +pub fn csharp_qualified_generics(depth: usize) -> String { + format!("class C {{ {}A field; }}\n", "A.".repeat(depth)) +} + +/// Elixir: multiple timeout clauses accepted by the grammar but rejected +/// by the compiler. An analyzer must still process them in linear time. +#[must_use] +pub fn elixir_many_timeouts(width: usize) -> String { + format!( + "receive(do: ({}x -> :a), after: ({}1 -> :t))\n", + "x -> :a; ".repeat(width), + "1 -> :t; ".repeat(width) + ) +} + /// Rust: `fn f() -> i32 { (((…1…))) }`. /// /// The paren chain is the shape that first surfaced both the `tokens` @@ -549,6 +566,24 @@ pub fn kotlin_short_interpolations(width: usize) -> String { format!("val s = \"{}\"\n", "$a ".repeat(width)) } +/// Elixir: `f(x, x, …, k: if(a, do: 1), k: if(a, do: 1), …)` — one call +/// with `width` positional arguments before `width` keyword pairs. +/// +/// The width shape for the keyword-section lookup #1574 added, which +/// asks of every pair whether its keyword list is the call's trailing +/// argument, and of every keyword-form clause which Call owns it. Its +/// first form re-read the argument list from its front per pair, +/// quadratic in a call with many of both: 5.4 s for 16 000 arguments in +/// a release build. The `if` in each value is the cognitive reading. +#[must_use] +pub fn elixir_mixed_arguments(width: usize) -> String { + format!( + "f({}{})\n", + "x, ".repeat(width), + vec!["k: if(a, do: 1)"; width].join(", ") + ) +} + /// Kotlin: `inline fun f(noinline p0: () -> Unit, …) {}` — one function /// with `width` modified parameters. /// @@ -963,6 +998,36 @@ const LINEAR_BOUND: f64 = 1.5; /// /// See `docs/development/benchmarking.md` for the per-probe table. pub const PROBES: &[Probe] = &[ + Probe { + name: "halstead/csharp-qualified-generics", + lang: LANG::Csharp, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Halstead], + reading: |m| m.halstead.total_operators(), + }, + render: csharp_qualified_generics, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "Each generic qualifier must be classified without climbing \ + the remaining qualified-name ancestry.", + }, + Probe { + name: "abc/elixir-many-timeouts", + lang: LANG::Elixir, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc, Metric::Cyclomatic], + reading: |m| m.abc.conditions_sum(), + }, + render: elixir_many_timeouts, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "Malformed receive timeout clauses must not each rescan every \ + message clause. The reading comes from the message clauses.", + }, Probe { name: "tokens/nested-paren", lang: LANG::Rust, @@ -1218,6 +1283,29 @@ pub const PROBES: &[Probe] = &[ ancestor, and each step now indexes the walker's \ ancestor chain rather than calling `Node::parent`.", }, + Probe { + name: "cognitive/elixir-mixed-arguments", + lang: LANG::Elixir, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Cognitive], + reading: |m| m.cognitive.cognitive_sum(), + }, + render: elixir_mixed_arguments, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1574's keyword-section lookup asks of every keyword \ + pair whether its list is the call's trailing argument \ + (cognitive), and of every keyword-form clause which \ + Call owns it (cyclomatic, ABC). Its first form found \ + the trailing argument by reading the argument list \ + from its front, once per pair: 0.10 s, 0.36 s, 1.4 s \ + and 5.4 s for 2 000 to 16 000 arguments in a release \ + build, exponent 1.94. The lookup now reads the list's \ + siblings off the walk's chain. The reading is one \ + `if` per pair.", + }, Probe { name: "nom/nested-fn", lang: LANG::Rust, diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index cd8ab8ce..394393b5 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -158,7 +158,7 @@ application would over-count. | Kotlin | A subject-less `when` arm scores through the predicate slot; a subject-ful arm scores per entry | A subject-less arm's condition (`when { x > 5 -> … }`) is an ordinary boolean expression, compiled as an `if` predicate, so the comparison inside it is already counted by the operator arms and a blanket per-entry increment double-counted it. A subject-ful arm (`when (x) { in 1..2 -> … }`) lists a pattern rather than an independent boolean expression, so the implicit `subject == pattern` is a decision nothing in the source spells and the entry itself pays for it (#1421). | | Ruby | A subject-less `case` scores each `when` pattern as an `\|\|` operand; a subject-ful `case` scores per clause | A subject-less clause (`case; when x > 5`) lists ordinary boolean expressions, evaluated like an `if` predicate, and several patterns (`when a, b`) are an implicit `\|\|`. Each pattern therefore scores once unless its own operator already counted it, so `when x > 5` scores 1 like `if x > 5`, and `when a, b` scores 2 like `if a \|\| b`. Before this, a blanket per-clause increment counted a comparison twice. A subject-ful clause (`case x; when 1`) matches each pattern with an implicit `pattern === x` that nothing in the source spells, so the clause pays once (#1453). | | Go | A tagless `switch` scores each case expression as an `\|\|` operand; a tagged `switch` scores per case | A tagless switch (`switch { case x > 5: }`, with or without an init statement) is `switch true`: each case expression is an ordinary boolean scored by the same slot an `if` predicate uses, and a list (`case a, b:`) is an implicit `\|\|`. So `case x > 5:` scores 1 like `if x > 5`, and `case a, b:` scores 2 like `if a \|\| b`. Before this, a blanket per-case increment counted a comparison twice. A tagged case (`switch x { case 1: }`) compares with an implicit `x == 1` that nothing in the source spells, so the case pays once; type-switch and `select` cases are unchanged (#1523). | -| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive`, of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). A bracketed keyword list in last position (`if(c, [do: 1, else: 2])`) is the same argument as the bare one and scores the same, as does a quoted key (`"else": 2`); a list anywhere else, or one with a non-pair item, holds no sections. A `Kernel.&&` / `Kernel.\|\|` remote call scores its operands as `a && b` / `a \|\| b` does, the piped value of `a \|> Kernel.\|\|(b)` included (#1574), and so do `Kernel.and` / `Kernel.or` (#1582). A `receive`'s `after` clause is its timeout, the path taken when no message clause matched in time, so it is the construct's default arm and scores nothing, as `_ ->` does in a `case` (#1581). | +| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive`, of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). A bracketed keyword list in last position (`if(c, [do: 1, else: 2])`) is the same argument as the bare one and scores the same, as does a quoted key (`"else": 2`); a list anywhere else, or one with a non-pair item, holds no sections. A `Kernel.&&` / `Kernel.\|\|` remote call scores its operands as `a && b` / `a \|\| b` does, the piped value of `a \|> Kernel.\|\|(b)` included (#1574), and so do `Kernel.and` / `Kernel.or` (#1582). A `receive` has one free arm, as a `case` does: its bare `_ ->` when the `do` section holds one, otherwise its `after` clause — the timeout, the path taken when no message clause matched in time — which then scores nothing; beside a bare `_ ->` the timeout is a path of its own and scores one (#1581). | | C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these brackets as an operator (a type-enclosing pair as one `<>` since #1559 and #1567), and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | | C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | @@ -530,11 +530,14 @@ or diverges, so it scores one in both variants, as its `if let` twin does. Before #1542 it scored nothing. Elixir's `receive` is a `case` over the mailbox, and modified scores -its clauses once, as it does a `case`'s. Its `after` clause is the -timeout: the path taken when no clause matched in time. That makes it -the construct's default arm, so it scores nothing in either variant, as -`_ ->` does in a `case`. A `receive` holding only an `after` is a timed -wait and scores nothing at all (#1581). +its clauses once, as it does a `case`'s. Like a `case` it has one free +arm: its bare `_ ->` when the `do` section holds one, otherwise its +`after` clause — the timeout, the path taken when no clause matched in +time. So `receive do {:a, y} -> y after 10 -> :t end` scores one +decision in standard, as the two-clause `case` does, and +`receive do {:a, y} -> y; _ -> 0 after 10 -> :t end` two: beside the +catch-all the timeout is a path of its own. A `receive` holding only an +`after` is a timed wait and scores nothing at all (#1581). ### Counting Rust's `?` operator @@ -760,7 +763,9 @@ tokens involved the same way they spell real operators: `java.util.List` bill their names as operands. The inferred type `var` does not, as C# bills nothing for its own. Since #1566 a Java class literal bills its parts rather than one operand spelled by its - whole text, since JLS §15.8.2 defines it as an expression rather than + whole text, since + [JLS §15.8.2](https://docs.oracle.com/javase/specs/jls/se21/html/jls-15.html#jls-15.8.2) + defines it as an expression rather than a literal: in `java.util.List.class` and `Foo[].class` each name and the `class` tail are operands and each `.` and `[]` an operator, as Groovy's `Foo.class` and Kotlin's and PHP's `Foo::class` bill theirs. diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index ab52fe4c..6f00bc58 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -148,6 +148,7 @@ Read it as follows. | `cyclomatic/nested-and` | Python | depth | shape control for the row below | linear | | `cyclomatic/nested-ternary` | Python | depth | `Node::parent_grandparent_match` (#1096) | linear | | `loc/nested-quote` | Elixir | depth | `loc`'s Elixir catch-all arm (#1096) | linear | +| `cognitive/elixir-mixed-arguments` | Elixir | width | the keyword-section lookup per pair and per keyword-form clause (#1574) | linear | | `nom/nested-attributed-fn` | Rust | depth | the `exclude_tests` outer-attribute scan (#1100) | linear | | `nom/wide-attributed-fn` | Rust | width | the same scan on the width axis (#1100) | linear | | `nom/deep-attribute-run` | Rust | depth | the same scan on the diagonal: depth and one attribute run together (#1446) | linear | @@ -430,7 +431,7 @@ one now, and `count/perl-error-tail-calls` prices it: |---|---|---|---|---| | `act_on_node` lends no cache | `count/perl-error-tail-calls` | 0.88 | 1.89 | 2.8 ms → 84.5 ms | -These three are the probes whose shape is meant not to parse: +These four are the probes whose shape is meant not to parse: `RECOVERY_PROBES` in `shapes.rs` exempts them from the parse check and asserts that each root is still an `ERROR`. diff --git a/docs/development/threshold_corpus.md b/docs/development/threshold_corpus.md index 76f4caa7..002ca8ae 100644 --- a/docs/development/threshold_corpus.md +++ b/docs/development/threshold_corpus.md @@ -74,7 +74,8 @@ changes therefore barely moved the default's footing. Their largest per-language Ruby did not move at all, and Tcl moved by under 0.1%. Ruby's low figures are a property of these two repositories, which hold their own methods short -with RuboCop's method-length and ABC cops. They are not a measurement gap. +with [RuboCop](https://docs.rubocop.org/rubocop/cops_metrics.html)'s method-length and ABC cops. +They are not a measurement gap. ## Corpus diff --git a/src/metrics/abc/elixir.rs b/src/metrics/abc/elixir.rs index c9d4b74a..02ee25d8 100644 --- a/src/metrics/abc/elixir.rs +++ b/src/metrics/abc/elixir.rs @@ -15,10 +15,10 @@ use super::{ }; use crate::lang_helpers::elixir::{ elixir_applying_operator, elixir_applying_remote_call, elixir_call_keyword, - elixir_kernel_operator_token, elixir_short_circuit_name, + elixir_kernel_short_circuit_name, }; use crate::metrics::elixir_decisions::{ - elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_keyword_section, + Section, elixir_arrow_is_with_clause, elixir_call_has_section, elixir_clause_is_decision, elixir_sole_unguarded_pattern, elixir_when_alternative, elixir_when_is_guard, }; use crate::*; @@ -88,9 +88,7 @@ fn elixir_binary_operator_is(node: &Node, accept: impl Fn(Elixir) -> bool) -> bo fn elixir_condition_scores_itself(expr: &Node, code: &[u8]) -> bool { use Elixir as E; - elixir_kernel_operator_token(expr, code) - .and_then(|name| elixir_short_circuit_name(&name, code)) - .is_some() + elixir_kernel_short_circuit_name(expr, code).is_some() || elixir_binary_operator_is(expr, |op| { matches!( op, @@ -187,13 +185,7 @@ fn elixir_count_clause(clause: &Node, code: &[u8], conditions: &mut f64) { // (`if b do … else … end`), or the keyword form's `else:` pair, bare // (`if b, do: …, else: …`) or bracketed (`if(b, [do: …, else: …])`). fn elixir_has_else(call: &Node, code: &[u8]) -> bool { - use Elixir as E; - - call.children().any(|child| match child.kind_id().into() { - E::DoBlock => child.children().any(|c| c.kind_id() == E::ElseBlock as u16), - _ if child.kind() == "arguments" => elixir_keyword_section(&child, code, "else").is_some(), - _ => false, - }) + elixir_call_has_section(call, code, Section::Else) } // The operands of a short-circuit `Kernel` remote call @@ -215,19 +207,14 @@ fn elixir_count_remote_chain<'a>( ancestors: Ancestors<'a, '_>, conditions: &mut f64, ) { - let Some(call) = elixir_applying_remote_call(op, code, ancestors) else { + let Some((call, call_ancestors)) = elixir_applying_remote_call(op, code, ancestors) else { return; }; - let mut up = ancestors - .iter(op) - .map(|(ancestor, _)| ancestor) - .skip_while(|ancestor| ancestor.id() != call.id()) - .skip(1); let score = |operand: &Node, conditions: &mut f64| { elixir_count_condition(operand, code, conditions); }; - if let Some(piped) = up - .next() + if let Some(piped) = call_ancestors + .parent(&call) .filter(|pipe| elixir_binary_operator_is(pipe, |op| op == Elixir::PIPEGT)) .and_then(|pipe| pipe.child_by_field_name("left")) { @@ -264,6 +251,12 @@ fn elixir_count_call(node: &Node, code: &[u8], stats: &mut Stats) { if !is_definition_or_directive { stats.branches += 1.; } + if keyword == Some("receive") + && let Some(timeout) = + crate::metrics::elixir_decisions::elixir_receive_timeout_clause(node, code) + { + elixir_count_clause(&timeout, code, &mut stats.conditions); + } // The predicate is the Call's first argument in both the block // (`if p do … end`) and keyword (`if p, do: …`) forms, and is a // slot (#1527). `arguments` carries five kind aliases at this diff --git a/src/metrics/cognitive/elixir.rs b/src/metrics/cognitive/elixir.rs index df715dd2..bd3a1b6d 100644 --- a/src/metrics/cognitive/elixir.rs +++ b/src/metrics/cognitive/elixir.rs @@ -15,7 +15,7 @@ use super::*; use crate::lang_helpers::elixir::{ elixir_applying_remote_call, elixir_short_circuit_kind, elixir_short_circuit_name, }; -use crate::metrics::elixir_decisions::{elixir_keyword_pair_section, elixir_receive_dispatches}; +use crate::metrics::elixir_decisions::{elixir_pair_section, elixir_receive_dispatches}; /// Whether a Call spelling `keyword` takes `try`'s sections — `else`, /// `rescue`, `catch` — in its keyword form: `try` itself, and the @@ -43,7 +43,7 @@ fn elixir_section<'a>( code: &'a [u8], ancestors: Ancestors<'a, '_>, ) -> Option { - let (key, call) = elixir_keyword_pair_section(pair, code, ancestors)?; + let (key, call) = elixir_pair_section(pair, code, ancestors)?; let keyword = elixir_call_keyword(&call, code)?; match key { "else" if matches!(keyword, "if" | "unless") || elixir_takes_try_sections(keyword) => { @@ -75,7 +75,7 @@ fn compute_elixir_remote_booleans<'a>( ancestors: Ancestors<'a, '_>, stats: &mut Stats, ) { - if let Some(call) = elixir_applying_remote_call(op, code, ancestors) + if let Some((call, _)) = elixir_applying_remote_call(op, code, ancestors) && let Some(key) = elixir_short_circuit_name(op, code) { stats.structural = @@ -211,9 +211,11 @@ impl Cognitive for ElixirCode { { increment_by_one(stats); } - // The keyword spellings of the sections above: - // `if(c, do: 1, else: 2)`, `try(do: f(), rescue: (…))`, bare - // or in brackets. + // The keyword spellings of the sections above, for the + // owners `elixir_section` names: `if(c, do: 1, else: 2)`, + // `try(do: f(), rescue: (…))`, bare or in brackets. A user + // macro's `foo(do: 1, else: 2)` scores nothing here while + // its `foo do 1 else 2 end` scores the `Else` arm above. E::Pair => match elixir_section(node, code, ancestors) { Some(ElixirSection::Else) => increment_by_one(stats), Some(ElixirSection::Handler) => increase_nesting(stats, &mut nesting), diff --git a/src/metrics/cyclomatic/elixir.rs b/src/metrics/cyclomatic/elixir.rs index 82751428..f7668684 100644 --- a/src/metrics/cyclomatic/elixir.rs +++ b/src/metrics/cyclomatic/elixir.rs @@ -10,7 +10,7 @@ use super::*; use crate::lang_helpers::elixir::{elixir_applying_operator, elixir_applying_remote_call}; use crate::metrics::elixir_decisions::{ elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_receive_dispatches, - elixir_when_is_guard, elixir_with_else_dispatches, + elixir_receive_timeout_clause, elixir_when_is_guard, elixir_with_else_dispatches, }; impl Cyclomatic for ElixirCode { @@ -170,6 +170,9 @@ impl Cyclomatic for ElixirCode { // (`elixir_clause_is_decision`). "receive" if elixir_receive_dispatches(node, code) => { stats.cyclomatic_modified += 1.; + if elixir_receive_timeout_clause(node, code).is_some() { + stats.cyclomatic += 1.; + } } // A `with`'s decisions are its `<-` clauses, // counted above in both tiers. What modified diff --git a/src/metrics/elixir_decisions.rs b/src/metrics/elixir_decisions.rs index 2b895302..2630a91a 100644 --- a/src/metrics/elixir_decisions.rs +++ b/src/metrics/elixir_decisions.rs @@ -342,12 +342,14 @@ fn elixir_is_first_clause<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> /// 10 -> :t end` reaches a matched message, any other message, and an /// empty mailbox timing out — three paths, the three-clause `case`'s /// two decisions. So the `after` clause is free only when the `do` -/// section has no bare catch-all ([`elixir_receive_after_is_decision`]). +/// section has no bare catch-all ([`elixir_receive_timeout_clause`]). /// It is never a dispatch in its own right: the compiler accepts /// exactly one `after` clause, and its head is a timeout value, not a /// pattern that can fail. No other construct puts a clause there, since /// `try`'s `after` takes a plain body, so the section is not among /// those [`DECISION_SECTION_KEYS`] names. +/// The owning Call scores the timeout once; scoring it per clause would +/// repeatedly scan the message section on compiler-invalid multi-timeout input. /// /// An anonymous fn's head clause is the closure's definition, not a /// dispatch: the closure opens its own function space whose base path @@ -389,9 +391,6 @@ pub(super) fn elixir_clause_is_decision<'a>( ) -> bool { use Elixir as E; - if let Some(receive) = elixir_after_section_receive(node, code, ancestors) { - return elixir_receive_after_is_decision(&receive, code); - } let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); let in_section = up.next().is_some_and(|parent| { matches!( @@ -442,43 +441,24 @@ fn elixir_is_catch_all(clause: &Node, code: &[u8]) -> bool { .is_some_and(|pattern| elixir_is_bare_wildcard(&pattern, code)) } -/// Whether a `receive`'s `after` clause is a decision: whether its `do` +/// A `receive`'s `after` clause when it is a decision: when its `do` /// section already holds the free bare `_ ->`, leaving the timeout as a /// path of its own. Without one the timeout is the construct's single /// free arm ([`elixir_clause_is_decision`]). -fn elixir_receive_after_is_decision(receive: &Node, code: &[u8]) -> bool { +/// Called once per receive, not once per timeout clause. +pub(super) fn elixir_receive_timeout_clause<'a>( + receive: &Node<'a>, + code: &'a [u8], +) -> Option> { + let timeout = receive.children().find_map(|child| { + elixir_section_holder(&child, code, Section::After)? + .children() + .find(|clause| clause.kind_id() == Elixir::StabClause as u16) + })?; elixir_section_has_clause(receive, code, Section::Do, |clause| { elixir_is_catch_all(clause, code) }) -} - -/// The `receive` Call whose `after` section holds `node` (a -/// `stab_clause`): the block form's `after_block` inside the Call's -/// `do_block`, or the keyword form's `after: (…)` block, bare or -/// bracketed. `None` for a clause anywhere else, and for an `after` -/// section of any other Call. -fn elixir_after_section_receive<'a>( - node: &Node<'a>, - code: &'a [u8], - ancestors: Ancestors<'a, '_>, -) -> Option> { - use Elixir as E; - - let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); - let call = match up.next().map(|parent| parent.kind_id().into()) { - Some(E::AfterBlock) => up - .next() - .filter(|do_block| do_block.kind_id() == E::DoBlock as u16) - .and_then(|_| up.next()), - Some(E::Block) => { - let pair = up.next()?; - elixir_pair_section(&pair, code, up) - .filter(|(key, _)| *key == "after") - .map(|(_, call)| call) - } - _ => None, - }?; - (elixir_call_keyword(&call, code) == Some("receive")).then_some(call) + .then_some(timeout) } /// Whether a `receive` Call dispatches over the mailbox: whether its @@ -494,47 +474,74 @@ pub(super) fn elixir_receive_dispatches(call: &Node, code: &[u8]) -> bool { elixir_section_has_clause(call, code, Section::Do, |_| true) } -/// A clause section [`elixir_section_has_clause`] reads. +/// A section of a Call, as [`elixir_section_holder`] finds it. #[derive(Clone, Copy)] -enum Section { +pub(super) enum Section { Do, Else, + After, } -/// Whether `call`'s `section` holds a `stab_clause` that `accept`s: the -/// block form's `do_block`, or the `else_block` inside it, or the -/// keyword form's `do:` / `else:` value, bare or bracketed -/// ([`elixir_trailing_keywords`]). The block form's `else_block` is the -/// `do_block`'s child, never the Call's. +impl Section { + /// The section's keyword-form spelling, without its colon. + const fn key(self) -> &'static str { + match self { + Self::Do => "do", + Self::Else => "else", + Self::After => "after", + } + } +} + +/// Whether `call`'s `section` holds a `stab_clause` that `accept`s. fn elixir_section_has_clause( call: &Node, code: &[u8], section: Section, accept: impl Fn(&Node) -> bool, ) -> bool { - use Elixir as E; - - let key = match section { - Section::Do => "do", - Section::Else => "else", - }; call.children().any(|child| { - let holder = match (child.kind_id().into(), section) { - (E::DoBlock, Section::Do) => Some(child), - (E::DoBlock, Section::Else) => child - .children() - .find(|c| c.kind_id() == E::ElseBlock as u16), - _ if child.kind() == ARGUMENTS => elixir_keyword_section(&child, code, key), - _ => None, - }; - holder.is_some_and(|holder| { + elixir_section_holder(&child, code, section).is_some_and(|holder| { holder .children() - .any(|clause| clause.kind_id() == E::StabClause as u16 && accept(&clause)) + .any(|clause| clause.kind_id() == Elixir::StabClause as u16 && accept(&clause)) }) }) } +/// Whether `call` has a `section`, in either spelling +/// ([`elixir_section_holder`]). +pub(super) fn elixir_call_has_section(call: &Node, code: &[u8], section: Section) -> bool { + call.children() + .any(|child| elixir_section_holder(&child, code, section).is_some()) +} + +/// The node holding the `section` that `child`, a child of a Call, +/// spells: the block form's `do_block`, or the `else_block` inside it, +/// or the keyword form's `do:` / `else:` value in the trailing keyword +/// list of the `arguments` `child` is, bare or bracketed +/// ([`elixir_trailing_keywords`]). The block form's `else_block` is the +/// `do_block`'s child, never the Call's. +fn elixir_section_holder<'a>( + child: &Node<'a>, + code: &'a [u8], + section: Section, +) -> Option> { + use Elixir as E; + + match (child.kind_id().into(), section) { + (E::DoBlock, Section::Do) => Some(*child), + (E::DoBlock, Section::Else) => child + .children() + .find(|c| c.kind_id() == E::ElseBlock as u16), + (E::DoBlock, Section::After) => child + .children() + .find(|c| c.kind_id() == E::AfterBlock as u16), + _ if child.kind() == ARGUMENTS => elixir_keyword_section(child, code, section.key()), + _ => None, + } +} + /// The keyword spellings of the clause sections whose clauses are /// decisions, which a `do … end` block spells as `do_block` / /// `else_block` / `rescue_block` / `catch_block`. Not `after`: see @@ -551,9 +558,10 @@ const DECISION_SECTION_KEYS: &[&str] = &["do", "else", "rescue", "catch"]; /// The grammar's key is a `keyword` token, which has no named parts, or a /// `quoted_keyword` around its `quoted_content` and any interpolations. fn elixir_pair_key<'a>(pair: &Node<'a>, code: &'a [u8]) -> Option<&'a str> { - let key = Some(pair) - .filter(|pair| pair.kind_id() == Elixir::Pair as u16) - .and_then(|pair| pair.child_by_field_name("key"))?; + if pair.kind_id() != Elixir::Pair as u16 { + return None; + } + let key = pair.child_by_field_name("key")?; let mut parts = key.children().filter(Node::is_named); match (parts.next(), parts.next()) { (None, _) => key @@ -578,7 +586,7 @@ fn elixir_pair_key<'a>(pair: &Node<'a>, code: &'a [u8]) -> Option<&'a str> { /// /// Matched by rule name: `keywords` carries two kind ids at this pin /// (grammar-dispatch §1). Comments are skipped as the extras they are. -pub(super) fn elixir_trailing_keywords<'a>(arguments: &Node<'a>) -> Option> { +fn elixir_trailing_keywords<'a>(arguments: &Node<'a>) -> Option> { let last = arguments.children().filter(is_operand).last()?; if last.kind() == KEYWORDS { return Some(last); @@ -593,11 +601,7 @@ pub(super) fn elixir_trailing_keywords<'a>(arguments: &Node<'a>) -> Option( - arguments: &Node<'a>, - code: &'a [u8], - key: &str, -) -> Option> { +fn elixir_keyword_section<'a>(arguments: &Node<'a>, code: &'a [u8], key: &str) -> Option> { elixir_trailing_keywords(arguments)? .children() .find(|pair| elixir_pair_key(pair, code) == Some(key))? @@ -607,44 +611,55 @@ pub(super) fn elixir_keyword_section<'a>( /// The section key of `pair` and the `Call` whose trailing keyword list /// holds it — `("else", if …)` for the `else:` of `if(c, do: 1, else: 2)` /// or of `if(c, [do: 1, else: 2])` — or `None` when `pair` is not a -/// keyword argument of a Call. `up` yields the ancestors above `pair`. +/// keyword argument of a Call. `ancestors` is `pair`'s chain. /// /// The climb is `keywords`, an optional bracketing `list`, `arguments`, -/// then the Call; the keyword list must be the one -/// [`elixir_trailing_keywords`] finds, so a `[do: x]` in any other -/// argument position, a map's keyword pairs and a nested list are not -/// sections. That identity check is also what rejects a list nested in -/// a list (`f([[do: x]])`), whose inner list the climb steps over. -fn elixir_pair_section<'a>( - pair: &Node<'a>, - code: &'a [u8], - mut up: impl Iterator>, -) -> Option<(&'a str, Node<'a>)> { - let key = elixir_pair_key(pair, code)?; - let keywords = up.next().filter(|keywords| keywords.kind() == KEYWORDS)?; - let arguments = up.find(|node| node.kind_id() != Elixir::List as u16)?; - up.next() - .filter(|call| { - call.kind_id() == Elixir::Call as u16 - && arguments.kind() == ARGUMENTS - && elixir_trailing_keywords(&arguments).map(|found| found.id()) - == Some(keywords.id()) - }) - .map(|call| (key, call)) -} - -/// [`elixir_pair_section`] for a `pair` the walker is visiting, its -/// ancestors read off the walker's chain. -pub(super) fn elixir_keyword_pair_section<'a>( +/// then the Call, and the keyword list must be the trailing one +/// [`elixir_trailing_keywords`] describes: no operand follows it among +/// the arguments, and a bracketing list holds nothing before it. So a +/// `[do: x]` in any other argument position, a map's keyword pairs and a +/// list nested in a list (`f([[do: x]])`, whose outer list is where the +/// `arguments` should be) are not sections. +/// +/// Each step reads one sibling off the chain rather than the argument +/// list from its front, so the pairs of one call cost it a scan each +/// only once: every pair asks this in cognitive, and every keyword-form +/// clause in cyclomatic and ABC, and a call holding thousands of +/// positional arguments beside thousands of pairs or clauses was +/// quadratic in them (5.4 s for 16 000 arguments in a release build; +/// `cognitive/elixir-mixed-arguments` prices the shape). +pub(super) fn elixir_pair_section<'a>( pair: &Node<'a>, code: &'a [u8], ancestors: Ancestors<'a, '_>, ) -> Option<(&'a str, Node<'a>)> { - elixir_pair_section( - pair, - code, - ancestors.iter(pair).map(|(ancestor, _)| ancestor), - ) + let mut up = ancestors.iter(pair); + let (keywords, keywords_ancestors) = up.next().filter(|(node, _)| node.kind() == KEYWORDS)?; + // The argument the keyword list is: itself, or the list bracketing it. + let (bracketed, (argument, argument_ancestors)) = match up.next()? { + (list, list_ancestors) if list.kind_id() == Elixir::List as u16 => { + (true, (list, list_ancestors)) + } + _ => (false, (keywords, keywords_ancestors)), + }; + let mut above = argument_ancestors.iter(&argument); + let (arguments, _) = above.next()?; + let (call, _) = above.next()?; + // The grammar ends a list with its `keywords`, so a bracketing list + // that holds nothing before them holds nothing else. + let is_section = arguments.kind() == ARGUMENTS + && call.kind_id() == Elixir::Call as u16 + && argument_ancestors + .siblings_after(&argument) + .all(|sibling| !is_operand(&sibling)) + && (!bracketed + || keywords_ancestors + .siblings_before(&keywords) + .all(|sibling| !is_operand(&sibling))); + is_section + .then(|| elixir_pair_key(pair, code)) + .flatten() + .map(|key| (key, call)) } /// The rule name of a keyword list (two kind ids at this pin). @@ -653,27 +668,41 @@ const KEYWORDS: &str = "keywords"; /// The rule name of an argument list (five kind ids at this pin). const ARGUMENTS: &str = "arguments"; -/// The `Call` whose `do` section holds `node` (a `stab_clause`): the -/// block form's `do_block` (whose parent is the call) or the keyword -/// form's `do: (…)` block, bare or bracketed (`block` → `pair` → the -/// climb [`elixir_pair_section`] makes). The caller asks the Call's -/// keyword (`cond`, `for`). +/// The `Call` whose `do` section holds `node` (a `stab_clause`), in +/// either spelling ([`elixir_clause_section`]). The caller asks the +/// Call's keyword (`cond`, `for`). fn elixir_do_section_call<'a>( node: &Node<'a>, code: &'a [u8], ancestors: Ancestors<'a, '_>, ) -> Option> { + elixir_clause_section(node, code, ancestors) + .filter(|(key, _)| *key == "do") + .map(|(_, call)| call) +} + +/// The section holding `node` (a `stab_clause`) and the Call it belongs +/// to: `("do", call)` under the block form's `do_block`, whose parent is +/// the Call, `("after", call)` under the `after_block` inside that +/// `do_block`, and the keyword form's `(key, call)` under a `do: (…)` / +/// `after: (…)` block, bare or bracketed (`block` → `pair` → the climb +/// [`elixir_pair_section`] makes). `None` for a clause anywhere else, +/// the block form's `else` / `rescue` / `catch` sections included, which +/// no caller asks the owner of. +fn elixir_clause_section<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> Option<(&'a str, Node<'a>)> { use Elixir as E; - let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); - match up.next().map(|parent| parent.kind_id().into()) { - Some(E::DoBlock) => up.next(), - Some(E::Block) => { - let pair = up.next()?; - elixir_pair_section(&pair, code, up) - .filter(|(key, _)| *key == "do") - .map(|(_, call)| call) - } + let mut up = ancestors.iter(node); + let (holder, _) = up.next()?; + let (next, next_ancestors) = up.next()?; + match holder.kind_id().into() { + E::DoBlock => Some(("do", next)), + E::AfterBlock if next.kind_id() == E::DoBlock as u16 => Some(("after", up.next()?.0)), + E::Block => elixir_pair_section(&next, code, next_ancestors), _ => None, } } diff --git a/src/metrics/elixir_spelling_parity_tests.rs b/src/metrics/elixir_spelling_parity_tests.rs index 2d8a08d8..12c5a6b5 100644 --- a/src/metrics/elixir_spelling_parity_tests.rs +++ b/src/metrics/elixir_spelling_parity_tests.rs @@ -14,9 +14,7 @@ //! `with … else` block form's cognitive score, decided in //! [`elixir_with_else_scores_like_case`] against `case`. -use std::cell::Cell; - -use crate::test_support::check_func_space_only; +use crate::test_support::metrics_verbatim; use crate::*; /// `[cyclomatic, cyclomatic_modified, abc conditions, cognitive]` sums of @@ -26,22 +24,17 @@ type Scores = [u64; 4]; #[cfg(feature = "elixir")] fn measure(source: &str) -> Scores { - let out = Cell::new([0; 4]); - check_func_space_only::( - &format!("{source}\n"), - "foo.exs", - &[Metric::Cyclomatic, Metric::Abc, Metric::Cognitive], - |space| { - let m = &space.metrics; - out.set([ - m.cyclomatic.cyclomatic_sum(), - m.cyclomatic.cyclomatic_modified_sum(), - m.abc.conditions_sum(), - m.cognitive.cognitive_sum(), - ]); - }, - ); - out.get() + let m = metrics_verbatim( + LANG::Elixir, + format!("{source}\n").as_bytes(), + MetricsOptions::default().with_only(&[Metric::Cyclomatic, Metric::Abc, Metric::Cognitive]), + ); + [ + m.cyclomatic.cyclomatic_sum(), + m.cyclomatic.cyclomatic_modified_sum(), + m.abc.conditions_sum(), + m.cognitive.cognitive_sum(), + ] } /// Asserts `twin` scores `want`, then that every spelling scores the same. diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 96f57395..25c6f9a5 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -7154,6 +7154,13 @@ end", ("alignas", "alignas(8) int a;\n"), ("_Alignas", "_Alignas(8) int a;\n"), ("__extension__", "__extension__ int a;\n"), + // The one place the grammar emits `__volatile__`: as a GNU `asm` + // qualifier, where the plain spelling bills too. + ( + "__volatile__", + "void f(void) { __asm__ __volatile__(\"nop\"); }\n", + ), + ("volatile", "void f(void) { asm volatile(\"nop\"); }\n"), ]; #[cfg(feature = "c")] diff --git a/src/metrics/nexits.rs b/src/metrics/nexits.rs index c2916cf4..307c0717 100644 --- a/src/metrics/nexits.rs +++ b/src/metrics/nexits.rs @@ -616,6 +616,7 @@ impl Exit for ElixirCode { mod tests { use crate::test_support::{ check_func_space_only_shim, check_metrics_only_shim, child_space, function_space, + metrics_verbatim, parse_named, }; use super::*; @@ -1679,12 +1680,13 @@ mod tests { /// The root space's `nexits` sum for the Perl file `source`. #[cfg(feature = "perl")] fn perl_nexits(source: &str) -> u64 { - let ast = Ast::parse(Source::new(LANG::Perl, source.as_bytes())).expect("perl is enabled"); - ast.metrics(MetricsOptions::default()) - .expect("metrics yields a root space") - .metrics - .nexits - .nexits_sum() + metrics_verbatim( + LANG::Perl, + source.as_bytes(), + MetricsOptions::default().with_only(&[Metric::Nexits]), + ) + .nexits + .nexits_sum() } /// Each auto-quoted `die` / `exit` key scores exactly as its quoted @@ -1764,7 +1766,7 @@ mod tests { use big_code_analysis_ast::node::WIDE_PARENT; let key = format!("my @r = ({}", "{\ndie => 1,\nexit,\n".repeat(12)); let twin = key.replace("die =>", "'die' =>"); - let ast = Ast::parse(Source::new(LANG::Perl, key.as_bytes())).expect("perl is enabled"); + let ast = parse_named(LANG::Perl, "foo.pl", &key); let root = ast.as_tree_sitter().root_node(); assert!( root.is_error() && root.child_count() > WIDE_PARENT, diff --git a/src/metrics/perl_keyword_key_tests.rs b/src/metrics/perl_keyword_key_tests.rs index 0b9b37b4..4b2d84f8 100644 --- a/src/metrics/perl_keyword_key_tests.rs +++ b/src/metrics/perl_keyword_key_tests.rs @@ -550,6 +550,12 @@ fn perl_spaced_dash_key_keeps_its_sign() { "f('-name' => 1, 'name' => 2);", "f(- name => 1, name => 2);", ), + // The slice from the sign carries the comment, so the key is + // still one operand apart from `name`. + ( + "f('-name' => 1, 'name' => 2);", + "f(- # c\n name => 1, name => 2);", + ), ] { let want = measure_with_branches(twin); assert_eq!( @@ -620,11 +626,14 @@ fn perl_file_test_key_stops_at_a_line_break() { /// including a subscript it is only part of; /// and a `-` applied to a real `not`. The file test itself is billed as /// nothing, since the grammar emits its `-X` as a hidden token. Every -/// value is `main`'s, measured before the fix. +/// value but the stray-comma row's is `main`'s, measured before the fix; +/// that row read the key `-foo` on `main`, whose `=>` lookup skipped +/// the `ERROR` the stray `,` recovers into, and reads as the grammar +/// does since #1572, as `(name, => 1)` does (see `is_key_position`). #[cfg(feature = "perl")] #[test] fn perl_file_test_outside_a_key_keeps_its_reading() { - let rows: [(&str, [u64; 4], Halstead); 11] = [ + let rows: [(&str, [u64; 4], Halstead); 12] = [ ("my $z = -f $x;", [2, 0, 0, 0], [6, 7, 3, 3]), ( "if (-e $x) { return 1; } return 0;", @@ -639,6 +648,8 @@ fn perl_file_test_outside_a_key_keeps_its_reading() { // Inside a subscript, but not the whole key. ("my %h; return $h{$a . -foo};", [2, 1, 0, 0], [7, 10, 5, 5]), ("f(-foo, 1);", [2, 2, 0, 0], [5, 5, 3, 4]), + // A stray `,` before the `=>` ends the key position. + ("my @c = (-foo, => 1);", [2, 1, 0, 0], [8, 8, 4, 4]), ("return -not $x;", [2, 0, 0, 0], [7, 7, 2, 2]), ("return $a - not $x;", [2, 0, 0, 0], [7, 8, 3, 3]), ]; @@ -1071,6 +1082,11 @@ fn perl_bareword_key_is_no_call_1580() { ("my %h = (1 - name => 2);", "my %h = (1 - 'name' => 2);", 0), ("new(name => $x);", "new('name' => $x);", 1), ("my %h = (Foo::Bar => 1);", "my %h = (Foo::Bar() => 1);", 1), + ( + "my %h = (-foo::bar => 1);", + "my %h = (-foo::bar() => 1);", + 1, + ), ("my %h = (name() => 1);", "my %h = (name(1) => 1);", 1), ("my %h = (k => name);", "my %h = ('k' => name());", 1), ]; @@ -1082,3 +1098,20 @@ fn perl_bareword_key_is_no_call_1580() { assert_eq!(branches, calls, "`{key}`: ABC disagrees"); } } + +#[cfg(feature = "perl")] +#[test] +fn perl_negated_qualified_calls_keep_their_operand_names() { + let source = "my %h = (-foo::bar => 1, -foo::baz => 2);"; + let ast = Ast::parse(Source::new(LANG::Perl, source.as_bytes())).expect("perl is enabled"); + assert_eq!(ast.count(&["call".to_owned()]).0, 2); + let metrics = ast + .metrics(MetricsOptions::default()) + .expect("metrics has a root"); + assert_eq!(metrics.metrics.abc.branches_sum(), 2); + assert_eq!(metrics.metrics.halstead.unique_operands(), 6); + assert_eq!( + ast.ops().expect("ops has a root").operands, + ["%h", "1", "2", "bar", "baz", "oo"] + ); +} diff --git a/src/ops.rs b/src/ops.rs index 9a6dac6e..6939e34c 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -1762,6 +1762,13 @@ mod tests { #[cfg(feature = "csharp")] #[test] fn csharp_misparsed_comparisons_stay_comparisons_1559() { + check_generic_pair( + LANG::Csharp, + "foo.cs", + "class A { void M() { F(Foo.a < b, c > d); } }\n", + &["()", ",", ".", ";", "<", "<>", ">", "class", "void", "{}"], + [10, 12], + ); check_generic_pair( LANG::Csharp, "foo.cs", From 1e1c0353d8ef846c526274535e1d08a3baf743f5 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:46:26 -0700 Subject: [PATCH 27/34] fix(gates): fold root mod cfg into the lang-gate compare `--compare` diffed per-test membership from each item's own cfg and its enclosing items', but never the cfg on the `mod` line that loads the file. A narrowed root gate in a `tests/*/main.rs` therefore dropped a whole suite from a single-language leg with nothing reporting it: #1567's deliberate drop of `kotlin` from `abc_halstead_bracket_parity` read as OK against main. Each file now inherits the conjoined cfg of the `mod` declarations that reach it, and `--compare` intersects test membership with it. The non-compare gate is unchanged. --- .claude/rules/testing.md | 3 + src/metrics/abc/slot_pays_tests.rs | 44 ++++ src/metrics/cognitive/word_operator_tests.rs | 14 ++ utils/check-test-lang-gates-test.py | 147 +++++++++++- utils/check-test-lang-gates.py | 225 ++++++++++++++----- 5 files changed, 380 insertions(+), 53 deletions(-) diff --git a/.claude/rules/testing.md b/.claude/rules/testing.md index bb317cb9..6ac196dc 100644 --- a/.claude/rules/testing.md +++ b/.claude/rules/testing.md @@ -651,6 +651,9 @@ tree at `ref` as well (`git archive` into a scratch directory), computes which single-language builds compile each test in each, and fails on a test that still exists but stopped being built somewhere. The probe set comes from the language table, so a new language extends it for free. +A test's membership includes the `cfg` on the `mod` line that loads its +file, so narrowing a root `mod` gate in a `tests/*/main.rs` is reported +too; before that, #1567's drop of `kotlin` from one went unseen. CI runs it per pull request against the base branch head — the PR's `base.sha`, not `git merge-base`, because the job checks out shallow and diff --git a/src/metrics/abc/slot_pays_tests.rs b/src/metrics/abc/slot_pays_tests.rs index 8e3575cf..1f605e0a 100644 --- a/src/metrics/abc/slot_pays_tests.rs +++ b/src/metrics/abc/slot_pays_tests.rs @@ -16,12 +16,56 @@ use crate::{LANG, Metric, MetricsOptions}; /// The placeholder a template marks its slot with. Not `@`, which /// Objective-C, PHP and Perl predicates spell. +#[cfg(any( + feature = "c", + feature = "cpp", + feature = "csharp", + feature = "elixir", + feature = "groovy", + feature = "irules", + feature = "java", + feature = "javascript", + feature = "kotlin", + feature = "lua", + feature = "mozcpp", + feature = "mozjs", + feature = "objc", + feature = "perl", + feature = "php", + feature = "python", + feature = "ruby", + feature = "rust", + feature = "tcl", + feature = "typescript", +))] const SLOT: &str = "PRED"; /// For each `(predicate, conditions, decisions)` row, replaces the /// template's `SLOT` with the predicate and asserts the function named `f` /// scores `conditions` ABC conditions and `decisions` cyclomatic /// decisions (`cyclomatic - 1`). +#[cfg(any( + feature = "c", + feature = "cpp", + feature = "csharp", + feature = "elixir", + feature = "groovy", + feature = "irules", + feature = "java", + feature = "javascript", + feature = "kotlin", + feature = "lua", + feature = "mozcpp", + feature = "mozjs", + feature = "objc", + feature = "perl", + feature = "php", + feature = "python", + feature = "ruby", + feature = "rust", + feature = "tcl", + feature = "typescript", +))] #[track_caller] fn assert_rows(lang: LANG, template: &str, rows: &[(&str, u64, u64)]) { assert!(template.contains(SLOT), "template has no predicate slot"); diff --git a/src/metrics/cognitive/word_operator_tests.rs b/src/metrics/cognitive/word_operator_tests.rs index 2c62cc8a..d0491baf 100644 --- a/src/metrics/cognitive/word_operator_tests.rs +++ b/src/metrics/cognitive/word_operator_tests.rs @@ -24,11 +24,25 @@ use crate::*; /// `(twin, mixed, words, expected)`: an unmixed expression, the same /// expression with some operators word-spelled, how many word-operator /// tokens `mixed` must carry, and the cognitive sum both must score. +#[cfg(any( + feature = "elixir", + feature = "irules", + feature = "perl", + feature = "php", + feature = "ruby", +))] type Row = (&'static str, &'static str, usize, u64); /// Scores every row inside `wrap`, after proving each mixed fixture /// carries its word operators — a fixture that silently parsed `and` as /// an identifier would otherwise pass while asserting nothing. +#[cfg(any( + feature = "elixir", + feature = "irules", + feature = "perl", + feature = "php", + feature = "ruby", +))] fn assert_rows( path: &str, wrap: impl Fn(&str) -> String, diff --git a/utils/check-test-lang-gates-test.py b/utils/check-test-lang-gates-test.py index cb8a2915..cf87fe99 100755 --- a/utils/check-test-lang-gates-test.py +++ b/utils/check-test-lang-gates-test.py @@ -796,6 +796,71 @@ def test_a_helper_called_from_a_narrower_helper_is_reported(self) -> None: [], ) + def test_a_language_bound_local_is_not_a_helper_call(self) -> None: + source = self.TWO_HOPS.replace( + " shared();\n }\n\n #[cfg(feature = \"python\")]", + " let shared = 1;\n assert_eq!(shared, 1);\n }\n\n #[cfg(feature = \"python\")]", + 1, + ) + self.assertEqual(_offending(source % 'feature = "python"'), []) + + def test_a_call_inherits_its_inner_guard(self) -> None: + source = self.TWO_HOPS.replace( + " shared();\n }\n\n #[cfg(feature = \"python\")]", + ' #[cfg(feature = "python")]\n shared();\n }\n\n #[cfg(feature = "python")]', + 1, + ) + self.assertEqual(_offending(source % 'feature = "python"'), []) + self.assertEqual( + _offending(self.TWO_HOPS % 'feature = "python"'), [("shared", ["rust"])] + ) + + def test_an_all_guard_keeps_a_matching_helper_gate(self) -> None: + source = ''' +#[cfg(test)] +mod tests { + #[cfg(all(feature = "python", feature = "rust"))] + fn shared() {} + #[test] + fn guarded_case() { + #[cfg(all(feature = "python", feature = "rust"))] + shared::(); + } +} +''' + self.assertEqual(_offending(source), []) + + def test_function_values_survive_a_local_in_another_scope(self) -> None: + source = ''' +#[cfg(test)] +mod tests { + #[cfg(feature = "python")] + fn shared() {} + #[cfg(feature = "python")] + #[test] + fn py() { check::(); shared(); } + #[cfg(feature = "rust")] + #[test] + fn rs() { + check::(); + consume(shared); + { let shared = 1; consume(shared); } + } +} +''' + self.assertEqual(_offending(source), [("shared", ["rust"])]) + mixed = source.replace( + "consume(shared);\n {", + '#[cfg(feature = "python")]\n shared();\n consume(shared);\n {', + 1, + ) + self.assertEqual(_offending(mixed), [("shared", ["rust"])]) + qualified = source.replace( + "consume(shared);\n { let shared = 1; consume(shared); }", + "let shared = 1;\n consume(self::shared);", + ) + self.assertEqual(_offending(qualified), [("shared", ["rust"])]) + def test_three_hops_through_a_helper_that_names_nothing(self) -> None: """The middle link names no language, so only its gate says python.""" source = """ @@ -1338,7 +1403,7 @@ def test_an_ungated_caller_strands_nothing_its_helper_never_needs(self) -> None: ) stray = _named(items, "stray") self.assertIsNone(gate.effective_predicate(items, stray)) - self.assertIn( + self.assertNotIn( stray.index, gate.callers_by_callee(items)[_named(items, "shared").index] ) self.assertEqual(gate.offenders(items, gate.resolve_needs(items)), []) @@ -2408,6 +2473,24 @@ def test_the_probe_set_comes_from_the_language_table(self) -> None: ) self.assertEqual(probes["--features rust"], frozenset({"rust"})) + def test_nested_file_modules_inherit_their_declaring_gate(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / "src" / "outer").mkdir(parents=True) + (root / "src" / "lib.rs").write_text("#[cfg(test)]\nmod outer;\n") + (root / "src" / "outer.rs").write_text( + '#[cfg(feature = "python")]\nmod suite;\n' + ) + (root / "src" / "outer" / "suite.rs").write_text( + "#[test]\nfn a_case() {}\n" + ) + sources = gate.rust_sources(root) + self.assertEqual( + gate.module_gates(root, sources)["src/outer/suite.rs"], + 'all(test, feature = "python")', + ) + self.assertIn("src/outer/suite.rs", gate.test_scope_files(root, sources)) + def test_membership_follows_the_gate(self) -> None: items = _scan( """ @@ -2496,6 +2579,68 @@ def git(*args: str) -> None: target.write_text("#[cfg(test)]\nmod tests {}\n") self.assertEqual(gate.compare_revisions("HEAD", root, table), []) + def test_a_root_mod_gate_that_narrowed_is_reported(self) -> None: + # The gate sits on the `mod` line in the crate root and the test + # file says nothing of it, so per-item membership alone cannot + # see the narrowing -- #1567 dropped `kotlin` from + # `tests/parity/main.rs` this way and `--compare` reported OK. + wide_root = ( + '#[cfg(any(feature = "python", feature = "rust"))]\n' + "mod suite;\n" + "mod ungated;\n" + ) + suite = """#[test] +fn a_case() { + for lang in LANG::into_enum_iter().filter(LANG::is_enabled) { + let _ = (LANG::Python, LANG::Rust, lang); + } +} +""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + crate = root / "tests" / "parity" + crate.mkdir(parents=True) + (root / "big-code-analysis-ast" / "src").mkdir(parents=True) + (root / "big-code-analysis-ast" / "src" / "langs.rs").write_text( + LANGS_FIXTURE + ) + main_rs = crate / "main.rs" + main_rs.write_text(wide_root) + (crate / "suite.rs").write_text(suite) + (crate / "ungated.rs").write_text(suite) + + env = { + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t", + "PATH": os.environ.get("PATH", ""), + "HOME": str(root), + } + for args in (["init", "-q"], ["add", "-A"], ["commit", "-q", "-m", "b"]): + subprocess.run( + ["git", *args], cwd=root, check=True, capture_output=True, env=env + ) + + table = gate.language_table(LANGS_FIXTURE) + self.assertEqual(gate.compare_revisions("HEAD", root, table), []) + + # Only the `mod` line narrows. `suite.rs` is byte-identical, + # and `ungated.rs`, loaded by an ungated `mod`, is the + # control: it must not be reported. + main_rs.write_text( + wide_root.replace( + 'any(feature = "python", feature = "rust")', 'feature = "python"' + ) + ) + self.assertEqual( + [ + (name, sorted(lost)) + for name, lost in gate.compare_revisions("HEAD", root, table) + ], + [("tests/parity/suite.rs::a_case", ["--features rust"])], + ) + class RepositoryTest(unittest.TestCase): def setUp(self) -> None: diff --git a/utils/check-test-lang-gates.py b/utils/check-test-lang-gates.py index 69ef86e2..25d4c5de 100755 --- a/utils/check-test-lang-gates.py +++ b/utils/check-test-lang-gates.py @@ -838,6 +838,9 @@ class Item: #: the dataclass stays comparable and printable. index: int in_test_scope: bool + #: Exact inner predicates at each call/reference, retained for implication checks. + call_guards: tuple[tuple[str, str], ...] = () + reference_guards: tuple[tuple[str, str], ...] = () def describe(self) -> str: return f"{self.path}:{self.line} {self.kind} {self.name}" @@ -860,16 +863,27 @@ def qualified_name(items: list[Item], item: Item) -> str: def membership( - per_file: dict[str, list[Item]], probes: dict[str, frozenset[str]] + per_file: dict[str, list[Item]], + probes: dict[str, frozenset[str]], + file_gates: dict[str, str] | None = None, ) -> dict[str, frozenset[str]]: - """Which probe builds compile each test, keyed by qualified name.""" + """Which probe builds compile each test, keyed by qualified name. + + ``file_gates`` (see `module_gates`) conjoins the ``cfg`` of the `mod` + line that loads each file onto every test in it. + """ languages = frozenset().union(*probes.values()) if probes else frozenset() result: dict[str, frozenset[str]] = {} - for items in per_file.values(): + for path, items in per_file.items(): + file_gate = (file_gates or {}).get(path) for item in items: if not (item.is_test_fn and item.in_test_scope): continue predicate = effective_predicate(items, item) + if file_gate is not None: + predicate = ( + file_gate if predicate is None else f"all({file_gate}, {predicate})" + ) if predicate is None: result[qualified_name(items, item)] = frozenset(probes) continue @@ -1007,6 +1021,35 @@ def _inner_cfg_extents( return extents +def _shadowed_references(body: str) -> set[int]: + """Offsets naming simple locals, respecting block scope and initializer use.""" + tokens = re.compile( + r"\blet\s+(?:mut\s+)?([a-z_][A-Za-z0-9_]*)|[{};]|\b[A-Za-z_][A-Za-z0-9_]*\b" + ) + scopes: list[set[str]] = [set()] + pending: list[set[str]] = [set()] + shadowed: set[int] = set() + for token in tokens.finditer(body): + name = token.group(1) + if name is not None: + shadowed.add(token.start(1)) + pending[-1].add(name) + elif token.group() == "{": + scopes.append(set()) + pending.append(set()) + elif token.group() == "}" and len(scopes) > 1: + scopes.pop() + pending.pop() + elif token.group() == ";": + scopes[-1].update(pending[-1]) + pending[-1].clear() + elif body[max(0, token.start() - 2):token.start()] == "::": + continue + elif any(token.group() in scope for scope in scopes): + shadowed.add(token.start()) + return shadowed + + def scan_source( source: str, path: str, @@ -1355,12 +1398,18 @@ def admits(disabled: frozenset[str], gates: list[str] = guarded[n]) -> bool: unguarded_references: set[str] = set() guarded_calls: list[tuple[str, frozenset[str]]] = [] guarded_references: list[tuple[str, frozenset[str]]] = [] - for pattern, unguarded_set, guarded_list in ( - (CALL_RE, unguarded_calls, guarded_calls), - (REFERENCE_RE, unguarded_references, guarded_references), + call_guards: list[tuple[str, str]] = [] + reference_guards: list[tuple[str, str]] = [] + shadowed = _shadowed_references(body) + for pattern, unguarded_set, guarded_list, sites in ( + (CALL_RE, unguarded_calls, guarded_calls, call_guards), + (REFERENCE_RE, unguarded_references, guarded_references, reference_guards), ): for hit in pattern.finditer(body): + if hit.start(1) in shadowed: + continue n = index + bisect.bisect_right(line_starts, hit.start()) - 1 + sites.append((hit.group(1), "all(" + ", ".join(guarded.get(n, ())) + ")")) languages = attribution[innermost[n]] if n in guarded else None if languages is None: unguarded_set.add(hit.group(1)) @@ -1389,15 +1438,17 @@ def admits(disabled: frozenset[str], gates: list[str] = guarded[n]) -> bool: direct=frozenset(direct), hardcoded=frozenset(hardcoded), hand_written=_hand_written_features(raw, attr_line), - calls=frozenset(m.group(1) for m in CALL_RE.finditer(body)), + calls=frozenset(name for name, _ in call_guards), unguarded_calls=frozenset(unguarded_calls), guarded_calls=tuple(guarded_calls), - references=frozenset(m.group(1) for m in REFERENCE_RE.finditer(body)), + references=frozenset(name for name, _ in reference_guards), unguarded_references=frozenset(unguarded_references), guarded_references=tuple(guarded_references), parent_index=parent_index, index=len(items), in_test_scope=in_test_scope, + call_guards=tuple(call_guards), + reference_guards=tuple(reference_guards), ) items.append(item) depth += line.count("{") - line.count("}") @@ -1445,53 +1496,113 @@ def rust_sources(root: pathlib.Path) -> list[pathlib.Path]: PATH_ATTR_RE = re.compile(r'#\[\s*path\s*=\s*"([^"]+)"\s*\]') -def test_scope_files(root: pathlib.Path, sources: list[pathlib.Path]) -> set[str]: - """Files that are wholly test modules by their declaration. +def _module_declarations( + root: pathlib.Path, sources: list[pathlib.Path] +) -> list[tuple[pathlib.Path, list[str], pathlib.Path]]: + """Every out-of-line ``mod x;`` as (declaring file, attributes, file). - `src/spaces_tests.rs` and its siblings hold nothing but tests, yet - carry no `#[cfg(test)]` of their own: the marker is on the - `#[cfg(test)] mod spaces_tests;` line in `lib.rs`. Reading that - declaration is what puts their 94 top-level tests in scope -- - without it they are taken for production code and left ungated, - which is 79 panics on a single-language leg. + Only declarations whose module file exists are returned. The + attributes are the outer ones stacked on the declaration; a file's + leading ``#![…]`` run is dropped, since it belongs to the file and + not to the first `mod` that happens to follow it. """ - scoped: set[str] = set() + found: list[tuple[pathlib.Path, list[str], pathlib.Path]] = [] for relative in sources: text = (root / relative).read_text(encoding="utf-8") lines = mask_dead(text).split("\n") raw = text.split("\n") index = 0 while index < len(lines): - if not lines[index].strip().startswith(("#[", "#![")): - index += 1 - continue - attrs, after = _join_attributes(lines, raw, index) + if lines[index].strip().startswith(("#[", "#![")): + attrs, after = _join_attributes(lines, raw, index) + attrs = [a for a in attrs if not a.startswith("#![")] + else: + attrs, after = [], index index = max(after, index + 1) if after >= len(lines): break declared = MOD_DECL_RE.match(lines[after]) if declared is None: continue - predicates = [cfg_predicate(a) for a in attrs] - if not any(p and predicate_requires_test(p) for p in predicates): - continue + index = after + 1 # `#[path = "…"]` renames the file the module comes from: # `src/spaces.rs` declares `#[path = "spaces_tests.rs"] mod # tests;`, so the name says `tests` and the file does not. override = PATH_ATTR_RE.search(" ".join(attrs)) name = declared.group(1) + module_dir = ( + relative.parent + if relative.name in {"lib.rs", "main.rs", "mod.rs"} + else relative.with_suffix("") + ) candidates = ( [relative.parent / override.group(1)] if override else [ - relative.parent / f"{name}.rs", - relative.parent / name / "mod.rs", + module_dir / f"{name}.rs", + module_dir / name / "mod.rs", ] ) - for candidate in candidates: - if (root / candidate).is_file(): - scoped.add(candidate.as_posix()) - return scoped + found.extend( + (relative, attrs, candidate) + for candidate in candidates + if (root / candidate).is_file() + ) + return found + + +def test_scope_files(root: pathlib.Path, sources: list[pathlib.Path]) -> set[str]: + """Files that are wholly test modules by their declaration. + + `src/spaces_tests.rs` and its siblings hold nothing but tests, yet + carry no `#[cfg(test)]` of their own: the marker is on the + `#[cfg(test)] mod spaces_tests;` line in `lib.rs`. Reading that + declaration is what puts their 94 top-level tests in scope -- + without it they are taken for production code and left ungated, + which is 79 panics on a single-language leg. + """ + return { + path for path, predicate in module_gates(root, sources).items() + if predicate_requires_test(predicate) + } + + +def module_gates(root: pathlib.Path, sources: list[pathlib.Path]) -> dict[str, str]: + """Each file's ``cfg`` as inherited from the `mod` lines that load it. + + A `#[cfg(any(feature = …))] mod suite;` in `tests/parity/main.rs` + gates every test in `tests/parity/suite.rs`, but nothing in that + file says so. `--compare` diffs per-test membership, so a narrowed + `mod` gate dropped a whole suite from a single-language leg with + nothing reporting it -- #1567's `kotlin` drop went unseen that way. + Gates conjoin down a chain of declarations; a file no gated `mod` + reaches is absent from the result and keeps its own membership. + """ + declared: dict[str, tuple[str, list[str]]] = {} + for parent, attrs, module in _module_declarations(root, sources): + predicates = [p for p in (cfg_predicate(a) for a in attrs) if p is not None] + declared[module.as_posix()] = (parent.as_posix(), predicates) + + resolved: dict[str, list[str]] = {} + + def chain(path: str, seen: frozenset[str]) -> list[str]: + if path in resolved: + return resolved[path] + if path not in declared or path in seen: + return [] + parent, own = declared[path] + parts = [*chain(parent, seen | {path}), *own] + resolved[path] = parts + return parts + + gates: dict[str, str] = {} + for path in declared: + parts = chain(path, frozenset()) + if parts: + gates[path] = ( + parts[0] if len(parts) == 1 else "all(" + ", ".join(parts) + ")" + ) + return gates def scan_tree(root: pathlib.Path, table: dict[str, str]) -> dict[str, list[Item]]: @@ -1742,6 +1853,9 @@ def reachable(item: Item, generic_too: bool) -> set[str]: continue if not _uses(item, items[callee_index], strict=False): continue + unguarded = item.unguarded_references + if name not in unguarded: + continue addition = needs[item.index] - needs[callee_index] if addition: needs[callee_index] |= addition @@ -2139,17 +2253,13 @@ def stranded_by_callers( ``predicate`` is the helper's own effective gate. An ungated helper never gets here: the union check has already reported it. - Only a build enabling something in ``required`` counts. The caller - link is the broad one, so a local that merely shares the helper's - name links an ungated caller to it, and that caller is in every - build; without the intersection the helper reads as stranded in all - of them, and the `any(required)` marker `--fix` writes can never - clear it. A real caller's languages are in ``required`` already, - since needs propagate over the same links, so the `any(…)` marker - `--fix` writes clears whatever this reports — when it can replace the - helper's gate. A gate `--fix` must stack on instead (an `all(…)`, a - `not(…)`, a non-language atom) can still conjoin it away; the fix - loop then reports non-convergence rather than looping. + Each edge conjoins the caller's gate with the inner predicates on + that particular use. A guarded call is absent when its guard is + false, even though the containing function still compiles. Only a + build enabling something in ``required`` counts, so reports remain + expressible by the derived helper marker. A gate `--fix` must stack + on instead (an `all(…)`, a `not(…)`, a non-language atom) can still + conjoin it away; the fix loop then reports non-convergence. A helper whose body names no language has an empty ``required`` and is never reported, even when its own gate excludes a caller's build @@ -2158,12 +2268,17 @@ def stranded_by_callers( """ stranded: set[str] = set() for index in callers.get(item.index, []): - caller_gate = effective_predicate(items, items[index]) - for disabled in _caller_builds(caller_gate): - if ( - caller_gate is None or evaluate_predicate(caller_gate, disabled) - ) and not evaluate_predicate(predicate, disabled): - stranded |= (KNOWN_FEATURES - disabled) & required + caller = items[index] + caller_gate = effective_predicate(items, caller) or "all()" + sites = caller.reference_guards + guards = {guard for name, guard in sites if name == item.name} or {"all()"} + for guard in guards: + call_gate = f"all({caller_gate}, {guard})" + for disabled in _caller_builds(call_gate): + if evaluate_predicate(call_gate, disabled) and not evaluate_predicate( + predicate, disabled + ): + stranded |= (KNOWN_FEATURES - disabled) & required return frozenset(stranded) @@ -2372,7 +2487,9 @@ def apply_fixes( # --------------------------------------------------------------------------- -def scan_revision(ref: str, root: pathlib.Path) -> dict[str, list[Item]]: +def scan_revision( + ref: str, root: pathlib.Path +) -> tuple[dict[str, list[Item]], dict[str, str]]: """Scan the tree as of ``ref``, extracted to a scratch directory. `git archive` rather than a worktree: it touches nothing in the @@ -2398,7 +2515,9 @@ def scan_revision(ref: str, root: pathlib.Path) -> dict[str, list[Item]]: table = language_table( (checkout / LANGS_RS.relative_to(REPO_ROOT)).read_text(encoding="utf-8") ) - return scan_tree(checkout, table) + return scan_tree(checkout, table), module_gates( + checkout, rust_sources(checkout) + ) finally: shutil.rmtree(checkout, ignore_errors=True) @@ -2413,7 +2532,7 @@ def compare_revisions( both wrong, only the previous revision says so. """ probes = probe_builds(table) - reference_tree = scan_revision(ref, root) + reference_tree, reference_gates = scan_revision(ref, root) # The same guard `main` applies to the working tree. A reference with # no tests in it makes every name miss the `after` side, so nothing # is ever reported and the check prints OK having compared nothing — @@ -2429,8 +2548,10 @@ def compare_revisions( "against. A scan root is missing from that revision, or is " "excluded from `git archive` by `.gitattributes`." ) - before = membership(reference_tree, probes) - after = membership(scan_tree(root, table), probes) + before = membership(reference_tree, probes, reference_gates) + after = membership( + scan_tree(root, table), probes, module_gates(root, rust_sources(root)) + ) lost: list[tuple[str, frozenset[str]]] = [] for name, was in sorted(before.items()): # Only a test that is still here. One that was deleted or renamed From 763ca0f894039c1fc94e7b4129592bf7bc0e07ad Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:51:15 -0700 Subject: [PATCH 28/34] docs: apply the review findings Re-measure the C, C++ and Objective-C halstead.effort rows at d361bc32: #1578, #1579 and `__volatile__` landed after the 18c28cac measurement and raise C-family effort. The median-language rate stays 2.20% (2.78% / 1.76% at 40k / 60k), so the default stands. Move the `(-foo, => 1)` row out of the "keeps its reading" test into a pin of the grammar misparse carrying FIXME(#1571), restructure the book's Halstead bullets so the declaration-modifier rule is stated once, and correct lesson 53, the #1574 and #1578 changelog entries, the vendor-spelling list and the probe table order. --- CHANGELOG.md | 36 +++- big-code-analysis-book/src/metrics.md | 192 ++++++++++-------- .../src/recipes/thresholds.md | 11 +- .../src/default_thresholds.rs | 2 +- docs/development/benchmarking.md | 2 +- docs/development/lessons_learned.md | 13 +- docs/development/threshold_corpus.md | 20 +- src/metrics/perl_keyword_key_tests.rs | 31 ++- 8 files changed, 185 insertions(+), 122 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd4f6ddc..b8645d26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,29 @@ for historical reference. ### Fixed +- **Bash cognitive complexity keeps a unary-test prefix in its logical + sequence.** `[ -n "$b" -a "$a" = 1 -a "$c" = 2 ]` now scores like + separate tests joined with `&&`; the `-o` / `||` spellings agree too. +- **Perl dash-key recovery preserves package-qualified calls.** + `-foo::bar => 1` remains a negated call rather than an auto-quoted key, + retaining its ABC branch, `find` / `count` match and Halstead name. +- **C# generic-name recovery is linear in qualification depth** and + preserves real generic qualifiers beside recovered comparisons. + **Elixir receive timeout analysis is linear in clause count**, including + malformed input with multiple timeout clauses. +- **Language-test gates preserve call-site conditions** and distinguish + local variables from same-named helpers. Module-gate comparison also + follows nested `foo.rs` → `foo/bar.rs` declarations. The JSON hook now + leaves Pyright's JSONC configuration to the Python type-check gate. + +- **ABC no longer counts a predicate twice through value-preserving + wrappers.** This covers JavaScript, Mozjs, TypeScript and TSX `await`, + C# `checked` / `unchecked` and `await`, Python `await`, Rust unsafe / + const blocks, casts and awaited async blocks, Kotlin expression labels + and annotations, and GNU C-family statement expressions and C++ + `co_await`. Wrapped comparisons and logical chains keep their existing + condition counts; wrapped bare values still pay for their boolean slot. + - **TypeScript and TSX bill their member modifiers as operators** (#1563): `public`, `private`, `protected`, `override`, `readonly`, `abstract` and `accessor`, as C# and Kotlin do, counting each keyword @@ -118,14 +141,13 @@ for historical reference. `__volatile__`, `__extension__`, …) and C's storage class `auto`. C++ and Mozcpp also bill `virtual`, `mutable`, `constinit`, `consteval`, `explicit`, `friend`, the access labels, `override`, `final` and the - `noexcept` - of an exception specification. Each counts once as its keyword, never - as the `storage_class_specifier` / `type_qualifier` / + `noexcept` of an exception specification. Each counts once as its + keyword, never as the `storage_class_specifier` / `type_qualifier` / `virtual_specifier` / `access_specifier` wrapper. `override` and `final` used as names stay operands, `extern "C"` keeps `"C"` an operand, and C++'s placeholder `auto`, attributes, MS calling - conventions and the Gecko `MOZ_*` macros are unchanged. Halstead and - MI scores of C-family code rise. + conventions and the Gecko `MOZ_*` macros are unchanged. Halstead volume + rises and MI scores of C-family code fall. - **Objective-C bills declaration modifiers as Halstead operators** (#1579). C's storage classes, function specifiers and type qualifiers bill as in C, so a `.m` scores a C declaration the same as a `.c` or @@ -194,7 +216,9 @@ for historical reference. `case`, already paid by the container (the block form scored 2 against the keyword form's 1). In `big-code-analysis-ast`, `elixir_remote_call_applies` is replaced by - `elixir_applying_remote_call`, which returns the applying call. + `elixir_applying_remote_call`, which returns the applying call and + its ancestor chain; `elixir_kernel_operator_token` is now + `elixir_kernel_short_circuit_name`. - **Elixir `receive` scores like `case`** (#1581): modified cyclomatic counts the construct once and cognitive complexity adds +nesting, in the block and keyword forms. A `receive` gets one free arm, as a diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 394393b5..050cb6f9 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -737,96 +737,108 @@ tokens involved the same way they spell real operators: slice, dict and annotation `:`, and Rust's annotation, field, bound and label `:`, which is a separate operator from `::`. Both languages left these out until #1486, so their Halstead scores rose with it. - Python's `class` keyword followed in #1550: it is now billed like - `def`, as Ruby, Kotlin, C# and PHP bill their `class`. Java, Groovy, - the JavaScript family, C, C++ and Objective-C followed in #1552 with - their type-declaration keywords: `class`, `struct`, `union`, `enum`, - `interface`, `record`, `trait`, `@interface`, `@implementation` and - `@protocol`. The `class` tail of Java's `C.class` is an operand, not - this keyword (#1566, below). - Kotlin's `interface` followed in #1556, and since #1558 Kotlin bills - every modifier, as C# does: the `enum` of - `enum class`, `data`, `sealed`, the visibility keywords, `open`, - `override`, `suspend`, `reified`, `out` and the rest each count once, - as the keyword and never as its `modifiers` wrapper. The same words - used as names bill no operator: where the grammar parses the name - (`val open = 1`) it is an operand, and where it cannot (`open = !open` - at the start of a statement, `val suspend = 1`, a parameter named - `vararg`) the word bills nothing. #1557 added C++'s `namespace`, `template`, `typename`, - `typedef` and `using` (C and Objective-C bill `typedef` too), so - `template ` and `template ` each bill their - keyword and score alike. It also added TypeScript's `type`, - `namespace`, `module`, `declare` and `global`, and made every - TypeScript type name an operand (`x: Foo`, `interface I`, - `Map`), as C#, Rust and the C family already did. Java followed - in #1560: `Foo f`, `implements I`, `List` and each segment of - `java.util.List` bill their names as operands. The inferred type - `var` does not, as C# bills nothing for its own. Since #1566 a Java - class literal bills its parts rather than one operand spelled by its - whole text, since +- **A declaration keyword is an operator.** Python's `class` is billed + like `def` since #1550, as Ruby, Kotlin, C# and PHP bill their + `class`. Java, Groovy, the JavaScript family, C, C++ and Objective-C + followed in #1552 with their type-declaration keywords: `class`, + `struct`, `union`, `enum`, `interface`, `record`, `trait`, + `@interface`, `@implementation` and `@protocol`. The `class` tail of + Java's `C.class` is an operand, not this keyword (see the class + literal below). Kotlin's `interface` followed in #1556. #1557 added + C++'s `namespace`, `template`, `typename`, `typedef` and `using` (C + and Objective-C bill `typedef` too), so `template ` and + `template ` each bill their keyword and score alike. It + also added TypeScript's `type`, `namespace`, `module`, `declare` and + `global`. #1561 made TypeScript's type-level operator keywords + operators, as their value-level siblings `as`, `typeof` and `extends` + already were: `satisfies`, `keyof`, `infer`, and the `is` and + `asserts` of a type predicate. #1564 made Kotlin's `typealias`, + `init`, `where`, `by`, `get` and `set` operators, as C# bills its + `get`, `set`, `init` and `where`. The `@get:` and `@set:` annotation + targets bill nothing, like `@field:`. The backing `field` in an + accessor is an operand, and the words used as names (`val by = 1`, + `x.get(i)`) stay operands. +- **A declaration modifier is an operator, keyed by its keyword and + never its wrapper node.** C#, Kotlin, Java, Groovy, TypeScript and + TSX, C, C++, Mozcpp and Objective-C bill every modifier once, as the + keyword and never as the node that wraps it: Kotlin's and Java's + `modifiers`, TypeScript's `accessibility_modifier` and + `override_modifier`, and the C family's `storage_class_specifier`, + `type_qualifier` and `virtual_specifier`. So the `const` of + `void f() const` and of `int * const p` each bill once. C# already + did; the others followed in #1558 (Kotlin), #1563 (TypeScript), #1565 + (Java and Groovy), #1578 (C, C++ and Mozcpp) and #1579 + (Objective-C). The exceptions: + - The same word used as a name stays an operand (`val open = 1`, + `let readonly = 1`, `obj.static`, `var sealed = 1`, + `int override = 1;`). Where Kotlin's grammar cannot parse the name + (`open = !open` at the start of a statement, `val suspend = 1`, a + parameter named `vararg`), the word bills nothing. + - The `static` of `import static` and `requires static` bills + nothing, as `import` does not. + - C++'s `auto` is a placeholder type and stays unbilled. Attributes, + the MS calling conventions, the Gecko `MOZ_*` macros and the Apple + SDK macros (`NS_INLINE`, `FOUNDATION_EXPORT`, `IBOutlet`) bill + nothing. + + The words each language bills: + + - Kotlin: the `enum` of `enum class`, `data`, `sealed`, the + visibility keywords, `open`, `override`, `suspend`, `reified`, + `out` and the rest. + - TypeScript and TSX: the member modifiers `public`, `private`, + `protected`, `override`, `readonly`, `abstract`, `accessor` and + `static`. JavaScript and Mozjs bill `static` too. Their grammars + lex `static get` followed by a newline as one `static get` token, + which bills as one operator, so that spelling scores one entry + where `static get y()` on one line bills `static` and `get`. + - Java and Groovy: the visibility keywords, `static`, `native`, + `strictfp`, `volatile`, `sealed` and `non-sealed`, where before + #1565 they billed only `abstract`, `final`, `transient`, + `synchronized` and `default`; plus the `permits` clause keyword + beside `extends` and `implements`. + - C, C++ and Mozcpp: every storage class, function specifier and + type qualifier: `static`, `extern`, `register`, `inline`, + `thread_local`, `const`, `volatile`, `restrict`, `_Atomic`, + `constexpr`, `alignas` and their vendor spellings (`__inline`, + `__forceinline`, `__restrict__`, `__thread`, `__volatile__`, + `__extension__`, …), plus C++'s `virtual`, `mutable`, `constinit`, + `consteval`, `explicit`, `friend`, the access labels, `override`, + `final` and `noexcept`. C bills its storage class `auto`. + `extern "C"` bills `extern` and keeps `"C"` an operand. + - Objective-C: C's list, plus the `@public`, `@private`, `@protected` + and `@package` ivar labels, the `@optional` and `@required` + protocol sections, the ownership qualifiers (`__weak`, `__strong`, + `__block`, …), the nullability qualifiers (`nullable`, + `_Nullable`, `__nullable`, …), `__kindof`, the `__covariant` and + `__contravariant` variance, and the `in`, `out`, `inout`, `bycopy`, + `byref` and `oneway` method-type qualifiers. A property attribute + (`nonatomic`, `copy`, `getter`) bills as its word; a `getter=` or + `setter=` value stays an operand. +- **A type name is an operand.** #1557 made every TypeScript type name + an operand (`x: Foo`, `interface I`, `Map`), as C#, Rust and + the C family already did. Java followed in #1560: `Foo f`, + `implements I`, `List` and each segment of `java.util.List` + bill their names as operands. The inferred type `var` does not, as + C# bills nothing for its own. Java's primitive types are operators + everywhere. Groovy, whose grammar parses a primitive type as a name, + bills it the same way since #1568: `int`, `boolean`, `void` and the + other primitives are operators keyed by their spelling, as a + declared type and as a class literal's receiver. Groovy reserves + these words, and they bill as operators; in the positions Groovy + nonetheless allows them as names (`x.int`, `[int: 1]`), they stay + operands. Java's `boolean` is listed as `boolean` rather than its + node kind `boolean_type`, so both languages report it alike. +- **A class literal bills its parts.** Since #1566 a Java class + literal bills its parts rather than one operand spelled by its whole + text, since [JLS §15.8.2](https://docs.oracle.com/javase/specs/jls/se21/html/jls-15.html#jls-15.8.2) - defines it as an expression rather than - a literal: in `java.util.List.class` and `Foo[].class` each name and - the `class` tail are operands and each `.` and `[]` an operator, as - Groovy's `Foo.class` and Kotlin's and PHP's `Foo::class` bill theirs. - The `int` of `int.class` and the `void` of `void.class` stay - operators, as Java bills them everywhere. Groovy, whose grammar - parses a primitive type as a name, bills it the same way since - #1568: `int`, `boolean`, `void` and the other primitives are - operators keyed by their spelling, as a declared type and as a class - literal's receiver. Groovy reserves these words, so as a member name - or a label (`x.int`, `[int: 1]`) they stay operands. Java's - `boolean` is listed as `boolean` rather than its node kind - `boolean_type`, so both languages report it alike. #1561 made - TypeScript's type-level operator keywords operators, as their - value-level siblings `as`, `typeof` and `extends` already were: - `satisfies`, `keyof`, `infer`, and the `is` and `asserts` of a type - predicate. #1563 made TypeScript's member modifiers operators, as C# - and Kotlin bill theirs: `public`, `private`, `protected`, `override`, - `readonly`, `abstract`, `accessor` and `static`, each the keyword and - never its `accessibility_modifier` or `override_modifier` wrapper. - JavaScript and Mozjs bill `static` too. Their grammars lex - `static get` followed by a newline as one `static get` token, which - bills as one operator, so that spelling scores one entry where - `static get y()` on one line bills `static` and `get`. The same words used as names - (`let readonly = 1`, `obj.static`) stay operands. #1564 made Kotlin's - `typealias`, `init`, `where`, `by`, `get` and `set` operators, as C# - bills its `get`, `set`, `init` and `where`. The `@get:` and `@set:` - annotation targets bill nothing, like `@field:`. The backing `field` - in an accessor is an operand, and the words used as names - (`val by = 1`, `x.get(i)`) stay operands. #1565 made Java and Groovy bill every - modifier, where they had billed only `abstract`, `final`, `transient`, - `synchronized` and `default`: the visibility keywords, `static`, - `native`, `strictfp`, `volatile`, `sealed` and `non-sealed`, plus the - `permits` clause keyword beside `extends` and `implements`. Java's are - the keywords, never their `modifiers` wrapper. The `static` of - `import static` and `requires static` bills nothing, as `import` does not, and - the words used as names (`var sealed = 1`) stay operands. #1578 made - C, C++ and Mozcpp bill every storage class, function specifier and - type qualifier: `static`, `extern`, `register`, `inline`, - `thread_local`, `const`, `volatile`, `restrict`, `_Atomic`, - `constexpr`, `alignas` and their vendor spellings (`__inline`, - `__restrict__`, `__thread`), plus C++'s `virtual`, `mutable`, - `constinit`, `consteval`, `explicit`, `friend`, the access labels, - `override`, `final` and `noexcept`. Each is the keyword, never its - `storage_class_specifier`, `type_qualifier` or `virtual_specifier` - wrapper, so the `const` of `void f() const` and of `int * const p` - each bill once. C bills its storage class `auto`. C++'s `auto` is a - placeholder type and stays unbilled. `extern "C"` bills `extern` and - keeps `"C"` an operand. Attributes, the MS calling conventions and - the Gecko `MOZ_*` macros bill nothing, and `override` and `final` - used as names (`int override = 1;`) stay operands. #1579 extended this - to Objective-C, which bills C's list as C does plus its own - modifiers: the `@public`, `@private`, `@protected` and `@package` - ivar labels, the `@optional` and `@required` protocol sections, the - ownership qualifiers (`__weak`, `__strong`, `__block`, …), the - nullability qualifiers (`nullable`, `_Nullable`, `__nullable`, …), - `__kindof`, the `__covariant` and `__contravariant` variance, and the - `in`, `out`, `inout`, `bycopy`, `byref` and `oneway` method-type - qualifiers. A property attribute (`nonatomic`, `copy`, `getter`) bills - as its word; a `getter=` or `setter=` value stays an operand. The - Apple SDK macros (`NS_INLINE`, `FOUNDATION_EXPORT`, `IBOutlet`) bill - nothing, as the `MOZ_*` macros do. + defines it as an expression rather than a literal: in + `java.util.List.class` and `Foo[].class` each name and the `class` + tail are operands and each `.` and `[]` an operator, as Groovy's + `Foo.class` and Kotlin's and PHP's `Foo::class` bill theirs. The + `int` of `int.class` and the `void` of `void.class` stay operators, + as Java bills them everywhere. - **Angle brackets that enclose types are one bracket pair.** The `<` and `>` around type arguments and type parameters count as one `<>` operator, billed at the `<` as `()` is at the `(`, and the `>` counts @@ -903,7 +915,7 @@ tokens involved the same way they spell real operators: `operator[]`, `operator()`, `operator""_x` and `operator co_await`. The `operator` keyword is an operator in both languages, as is C++'s `co_await` in an expression. C++'s `[]` and `()` names are tokens of - their own, so `bca ops` lists `operator[]`'s `[]` beside a + their own, so `bca ops` lists `operator[]` beside a subscript's `[]` as a second operator, as Ruby does for `def [](i)` (#1296). C++'s coroutine keywords `co_return` and `co_yield` are operators as `return` is (#1547). diff --git a/big-code-analysis-book/src/recipes/thresholds.md b/big-code-analysis-book/src/recipes/thresholds.md index 05dc6ae2..44f25940 100644 --- a/big-code-analysis-book/src/recipes/thresholds.md +++ b/big-code-analysis-book/src/recipes/thresholds.md @@ -94,8 +94,8 @@ tighten, not evidence that you should. | Language | `cognitive` | `cyclomatic` | `abc` | `halstead.effort` | `loc.ploc` | | --- | --- | --- | --- | --- | --- | | Bash | 35 | 25 | 60 | 44000 | (thin sample) | -| C | 50 | 30 | 60 | 180000 | 3500 | -| C++ | 14 | 10 | 25 | 50000 | (thin sample) | +| C | 50 | 30 | 60 | 190000 | 3500 | +| C++ | 14 | 10 | 25 | 54000 | (thin sample) | | C# | 4 | 4 | 13 | 44000 | 700 | | Elixir | 5 | 7 | 20 | 15000 | 1500 | | Go | 25 | 17 | 45 | 89000 | 1500 | @@ -104,7 +104,7 @@ tighten, not evidence that you should. | JavaScript | 20 | 15 | 35 | 170000 | (thin sample) | | Kotlin | 10 | 8 | 20 | 18000 | 300 | | Lua | 35 | 20 | 40 | 82000 | 800 | -| Objective-C | 18 | 12 | 35 | 73000 | (thin sample) | +| Objective-C | 18 | 12 | 35 | 77000 | (thin sample) | | Perl | 35 | 25 | 45 | 41000 | 1500 | | PHP | 10 | 9 | 25 | 26000 | 900 | | Python | 20 | 13 | 25 | 42000 | 700 | @@ -122,7 +122,10 @@ The `halstead.effort` column comes from a different measurement than the rest of re-measured on 2026-10-10, after the Halstead vocabulary changes in [#1486](https://github.com/dekobon/big-code-analysis/issues/1486) through [#1573](https://github.com/dekobon/big-code-analysis/issues/1573), on a rebuilt corpus of 41 -repositories that is comparable to the original but not identical to it. The repository list and +repositories that is comparable to the original but not identical to it. The C, C++, and +Objective-C rows were re-measured on 2026-10-15 to include the declaration modifiers of +[#1578](https://github.com/dekobon/big-code-analysis/issues/1578) and +[#1579](https://github.com/dekobon/big-code-analysis/issues/1579). The repository list and commits are recorded in [`docs/development/threshold_corpus.md`](https://github.com/dekobon/big-code-analysis/blob/main/docs/development/threshold_corpus.md). Several rows moved by a factor of two to five, and almost all of that movement came from the change diff --git a/big-code-analysis-cli/src/default_thresholds.rs b/big-code-analysis-cli/src/default_thresholds.rs index 47e1be92..30a46887 100644 --- a/big-code-analysis-cli/src/default_thresholds.rs +++ b/big-code-analysis-cli/src/default_thresholds.rs @@ -107,7 +107,7 @@ pub(crate) const DEFAULT_THRESHOLDS: &[DefaultThreshold] = &[ "No published threshold exists for effort, so this one is purely", "percentile-derived: it fires on ~2% of functions in the median", "language. It is also the most language-dependent limit in the", - "table — per-language p90 effort ranges from 3k (Ruby) to 60k", + "table — per-language p90 effort ranges from 3k (Ruby) to 64k", "(C) — so it is the first one worth overriding per language.", ], }, diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index 6f00bc58..17f87c38 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -148,7 +148,6 @@ Read it as follows. | `cyclomatic/nested-and` | Python | depth | shape control for the row below | linear | | `cyclomatic/nested-ternary` | Python | depth | `Node::parent_grandparent_match` (#1096) | linear | | `loc/nested-quote` | Elixir | depth | `loc`'s Elixir catch-all arm (#1096) | linear | -| `cognitive/elixir-mixed-arguments` | Elixir | width | the keyword-section lookup per pair and per keyword-form clause (#1574) | linear | | `nom/nested-attributed-fn` | Rust | depth | the `exclude_tests` outer-attribute scan (#1100) | linear | | `nom/wide-attributed-fn` | Rust | width | the same scan on the width axis (#1100) | linear | | `nom/deep-attribute-run` | Rust | depth | the same scan on the diagonal: depth and one attribute run together (#1446) | linear | @@ -161,6 +160,7 @@ Read it as follows. | `halstead/kotlin-short-interpolations` | Kotlin | width | the `$name` test's previous-token lookup in one string (#1584) | linear | | `halstead/kotlin-noinline-params` | Kotlin | width | the modifier recovery guard's next-token lookup (#1558) | linear | | `abc/perl-pattern-list` | Perl | width | ABC's pattern-role climb through `( … )` (#1467) | linear | +| `cognitive/elixir-mixed-arguments` | Elixir | width | the keyword-section lookup per pair and per keyword-form clause (#1574) | linear | | `cyclomatic/cpp-plus-chain` | C++ | depth | shape control for the three rows below | linear | | `cyclomatic/cpp-and-chain` | C++ | depth | the `&&` / `||` applied-operator gate (#1533) | linear | | `cognitive/cpp-and-chain` | C++ | depth | the boolean-sequence arm on the same chain (#1533) | linear | diff --git a/docs/development/lessons_learned.md b/docs/development/lessons_learned.md index dd42a995..b22b4611 100644 --- a/docs/development/lessons_learned.md +++ b/docs/development/lessons_learned.md @@ -2116,9 +2116,9 @@ that look interchangeable can put the `;` at different indices, and an empty-slot regression test written against the wrong one passes vacuously under the old code. The same rule holds upward: a fixed-depth parent climb (`up.nth(2)`) is positional addressing of an ancestor. -Climb by role, stepping over an optional wrapper, and verify the -identity top-down — that the node reached is the one the construct's -own accessor returns. +Climb by role, stepping over at most one optional wrapper, and verify +the position from the siblings along the chain you climbed rather than +by re-reading the construct from its front. Each language's grammar makes a different choice about which slots are optional, so the bug is per-language and per-statement-kind, not @@ -2161,9 +2161,10 @@ between `keywords` and `arguments`, so the climb from a pair landed on `with` lost its modified-cyclomatic `else` tier, `for … reduce:` scored its accumulator head as a decision, `cond` lost its free `true ->`, and keyword-form `if` / `unless` lost ABC's `else` condition. The fix climbs -by role over the optional `list` and accepts the node only when -`elixir_trailing_keywords` returns the same pair from the top — which is -also what rejects a non-trailing or nested list. +by role over at most one bracketing `list` and decides from the chain's +siblings: no operand may follow the argument, and in a bracketed list no +operand may precede `keywords`. That is also what rejects a non-trailing +or nested list. --- diff --git a/docs/development/threshold_corpus.md b/docs/development/threshold_corpus.md index 002ca8ae..daa801b9 100644 --- a/docs/development/threshold_corpus.md +++ b/docs/development/threshold_corpus.md @@ -15,9 +15,13 @@ exclusions, so it is comparable to #1140's, but it is not the same set of reposi ## Method -- Measured 2026-10-10 with a release `bca` built from `fix/batch-2026-10-10` at `18c28cac`, with - every vocabulary change applied. The same corpus was also measured with the `v2.3.0` release, which - predates all of them, to separate the metric change from the corpus change. +- Measured 2026-10-10 with a release `bca` built from `fix/batch-2026-10-10` at `18c28cac`. The C, + C++, and Objective-C rows were re-measured on 2026-10-15 at `d361bc32`, the full branch, because + three later changes bill more C-family declaration modifiers: #1578 (C, C++, Mozcpp), #1579 + (Objective-C), and `__volatile__`. No other language's vocabulary changed after `18c28cac`, and the + C#, Elixir, Perl, and Ruby rows reproduce exactly at `d361bc32`. The same corpus was also measured + with the `v2.3.0` release, which predates all of these changes, to separate the metric change from + the corpus change. - Each repository was cloned with `git clone --depth 1` at its default branch. - `bca metrics --no-config -O json -p ` ran once per repository. The `-I` globs restricted it to the language's own extensions: `*.sh` (Bash-it also `*.bash`), `*.c`, `*.cpp *.cc *.h *.hpp`, `*.cs`, @@ -46,9 +50,9 @@ exclusions, so it is comparable to #1140's, but it is not the same set of reposi | Language | Functions | p50 | p90 | p97.5 | % > 50,000 | p97.5 at v2.3.0 | % > 50,000 at v2.3.0 | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | | Bash | 1,803 | 803 | 8,699 | 44,195 | 2.1 | 44,195 | 2.1 | -| C | 6,361 | 5,582 | 60,359 | 182,534 | 11.8 | 173,763 | 11.3 | +| C | 6,361 | 6,049 | 63,982 | 190,936 | 12.5 | 173,763 | 11.3 | | C# | 6,214 | 375 | 6,914 | 43,715 | 2.3 | 43,957 | 2.3 | -| C++ | 2,491 | 600 | 12,005 | 49,665 | 2.5 | 49,790 | 2.4 | +| C++ | 2,491 | 708 | 13,327 | 53,558 | 2.8 | 49,790 | 2.4 | | Elixir | 4,535 | 460 | 4,398 | 15,121 | 0.4 | 15,121 | 0.4 | | Go | 9,454 | 1,310 | 21,802 | 89,276 | 4.7 | 89,276 | 4.7 | | Groovy | 12,529 | 221 | 5,275 | 25,353 | 1.4 | 23,038 | 1.3 | @@ -56,7 +60,7 @@ exclusions, so it is comparable to #1140's, but it is not the same set of reposi | JavaScript | 31,030 | 1,191 | 32,475 | 174,060 | 7.2 | 174,108 | 7.3 | | Kotlin | 6,682 | 428 | 4,945 | 18,334 | 0.6 | 17,413 | 0.6 | | Lua | 4,562 | 2,673 | 27,416 | 81,544 | 5.1 | 81,544 | 5.1 | -| Objective-C | 1,209 | 1,215 | 15,441 | 72,897 | 4.1 | 75,726 | 4.1 | +| Objective-C | 1,209 | 1,351 | 16,764 | 76,893 | 4.4 | 75,726 | 4.1 | | Perl | 1,760 | 640 | 12,537 | 41,205 | 2.0 | 43,862 | 2.2 | | PHP | 20,296 | 625 | 6,857 | 26,333 | 1.2 | 26,333 | 1.2 | | Python | 10,264 | 1,030 | 10,904 | 41,983 | 2.0 | 28,535 | 1.2 | @@ -68,7 +72,9 @@ exclusions, so it is comparable to #1140's, but it is not the same set of reposi At the shipped limit of 50,000, the median language flags 2.20% of its functions, inside the 1–3% band the sizing rule asks for, so the default stands. Candidate limits for comparison: 40,000 gives -2.78% and 60,000 gives 1.76%. The `v2.3.0` binary gives 2.23% on the same corpus. The vocabulary +2.78% and 60,000 gives 1.76%. The C-family re-measure moved none of these three figures, +because C, C++, and Objective-C rank above the middle pair at every limit before and after it. +The `v2.3.0` binary gives 2.23% on the same corpus. The vocabulary changes therefore barely moved the default's footing. Their largest per-language effect is Python (p97.5 28,535 → 41,983), followed by Java (+17%) and Groovy (+10%). Bash, Elixir, Go, Lua, PHP, and Ruby did not move at all, and Tcl moved by under 0.1%. diff --git a/src/metrics/perl_keyword_key_tests.rs b/src/metrics/perl_keyword_key_tests.rs index 4b2d84f8..fa2b5df0 100644 --- a/src/metrics/perl_keyword_key_tests.rs +++ b/src/metrics/perl_keyword_key_tests.rs @@ -626,14 +626,11 @@ fn perl_file_test_key_stops_at_a_line_break() { /// including a subscript it is only part of; /// and a `-` applied to a real `not`. The file test itself is billed as /// nothing, since the grammar emits its `-X` as a hidden token. Every -/// value but the stray-comma row's is `main`'s, measured before the fix; -/// that row read the key `-foo` on `main`, whose `=>` lookup skipped -/// the `ERROR` the stray `,` recovers into, and reads as the grammar -/// does since #1572, as `(name, => 1)` does (see `is_key_position`). +/// value is `main`'s, measured before the fix. #[cfg(feature = "perl")] #[test] fn perl_file_test_outside_a_key_keeps_its_reading() { - let rows: [(&str, [u64; 4], Halstead); 12] = [ + let rows: [(&str, [u64; 4], Halstead); 11] = [ ("my $z = -f $x;", [2, 0, 0, 0], [6, 7, 3, 3]), ( "if (-e $x) { return 1; } return 0;", @@ -648,8 +645,6 @@ fn perl_file_test_outside_a_key_keeps_its_reading() { // Inside a subscript, but not the whole key. ("my %h; return $h{$a . -foo};", [2, 1, 0, 0], [7, 10, 5, 5]), ("f(-foo, 1);", [2, 2, 0, 0], [5, 5, 3, 4]), - // A stray `,` before the `=>` ends the key position. - ("my @c = (-foo, => 1);", [2, 1, 0, 0], [8, 8, 4, 4]), ("return -not $x;", [2, 0, 0, 0], [7, 7, 2, 2]), ("return $a - not $x;", [2, 0, 0, 0], [7, 8, 3, 3]), ]; @@ -662,6 +657,28 @@ fn perl_file_test_outside_a_key_keeps_its_reading() { } } +/// `(-foo, => 1)` pins the grammar's misparse, not Perl's reading. +/// +/// Perl reads `-foo` here as the string `-foo`: `perl -e 'my @c = +/// (-foo, => 1); print "@c"'` prints `-foo 1`. tree-sitter-perl instead +/// parses `-foo` as a `file_handle_operator` applied to the bareword +/// `oo`, and the stray `,` ends the key position, so the `4, 4` +/// operands below come from billing `oo`. `main` read the key `-foo`, +/// because its `=>` lookup skipped the `ERROR` the stray `,` recovers +/// into; since #1572 the shape reads as the grammar does, as +/// `(name, => 1)` does (see `is_key_position`). The value is pinned so +/// a change to it is seen, not endorsed as correct. +// FIXME(#1571): bill `-foo` as the one string operand Perl reads. +#[cfg(feature = "perl")] +#[test] +fn perl_dash_key_before_stray_comma_pins_the_grammar_misparse_1571() { + assert_eq!( + measure_with_branches("my @c = (-foo, => 1);"), + ([2, 1, 0, 0], [8, 8, 4, 4]), + "the grammar's `oo` reading moved; re-derive against #1571" + ); +} + /// `(quoted_twin, bareword_key, [cyclomatic, branches, conditions, /// cognitive], halstead)` rows for /// `perl_bareword_key_scores_like_its_quoted_twin`. Every value is the From 655e2ba4a44b88f53a6aa461cc1f70e78d03e0ca Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:44:07 -0700 Subject: [PATCH 29/34] fix(abc): preserve condition counts through transparent wrappers --- src/metrics/abc/cpp.rs | 8 +- src/metrics/abc/csharp.rs | 3 +- src/metrics/abc/js_family.rs | 3 +- src/metrics/abc/kotlin.rs | 6 + src/metrics/abc/python.rs | 2 +- src/metrics/abc/rust.rs | 26 ++++- src/metrics/abc/slot_pays_tests.rs | 179 ++++++++++++++++++++++++++++- 7 files changed, 219 insertions(+), 8 deletions(-) diff --git a/src/metrics/abc/cpp.rs b/src/metrics/abc/cpp.rs index 6753adf3..5093860e 100644 --- a/src/metrics/abc/cpp.rs +++ b/src/metrics/abc/cpp.rs @@ -11,7 +11,7 @@ use super::{ Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, - count_negated_operand, is_operand, wrapped_operand, + count_negated_operand, is_operand, last_operand, wrapped_operand, }; use crate::metrics::cyclomatic::cpp_operator_is_applied; use crate::*; @@ -51,7 +51,11 @@ fn cpp_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { .is_some_and(|op| accept.contains(&op.kind())) }; let value = match node.kind() { - "parenthesized_expression" => wrapped_operand(node), + "parenthesized_expression" | "co_await_expression" => wrapped_operand(node), + // A GNU statement expression yields only its final expression statement. + "compound_statement" => last_operand(node) + .filter(|statement| statement.kind() == "expression_statement") + .and_then(|statement| wrapped_operand(&statement)), "condition_clause" | "cast_expression" | "declaration" => node.child_by_field_name("value"), "initializer_list" => sole_operand(node), "comma_expression" => node.child_by_field_name("right"), diff --git a/src/metrics/abc/csharp.rs b/src/metrics/abc/csharp.rs index 5ceee2fb..99a8812f 100644 --- a/src/metrics/abc/csharp.rs +++ b/src/metrics/abc/csharp.rs @@ -78,8 +78,9 @@ fn csharp_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { use Csharp::*; match node.kind_id().into() { - // `(expr)` — the inner expression follows the `(` token. + // Parentheses, checked/unchecked and await preserve the value's condition. csharp_paren_expr_kinds!() => wrapped_operand(node).map(|o| (o, false)), + CheckedExpression | AwaitExpression => wrapped_operand(node).map(|o| (o, false)), // `!expr` — the operand follows the operator token. Seven other // prefix operators (`++ -- + - ~ & ^`) share this kind, as does // the `*` of a pointer indirection the grammar aliases onto it; diff --git a/src/metrics/abc/js_family.rs b/src/metrics/abc/js_family.rs index ec0aba7b..cc4de47e 100644 --- a/src/metrics/abc/js_family.rs +++ b/src/metrics/abc/js_family.rs @@ -21,7 +21,7 @@ use crate::*; // (grammar-dispatch §1: TypeScript alone emits two ids for // `parenthesized_expression` and for `unary_expression`). // -// One step of the value peel (see `PeelStep`). A parenthesis, a TS type +// One step of the value peel (see `PeelStep`). A parenthesis, an await, a TS type // cast (`x as T`, `x satisfies T`, `x`) and a non-null assertion // (`x!`) evaluate to their operand; a comma sequence to its last // operand; an assignment to its `right` value, so `if (y = x > 1)` tests @@ -34,6 +34,7 @@ use crate::*; fn js_family_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { match node.kind() { "parenthesized_expression" + | "await_expression" | "as_expression" | "satisfies_expression" | "non_null_expression" => wrapped_operand(node).map(|o| (o, false)), diff --git a/src/metrics/abc/kotlin.rs b/src/metrics/abc/kotlin.rs index 04154e4f..4059e485 100644 --- a/src/metrics/abc/kotlin.rs +++ b/src/metrics/abc/kotlin.rs @@ -94,6 +94,12 @@ fn kotlin_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { match node.kind_id().into() { // `(expr)` — the inner expression follows the `(` token. ParenthesizedExpression => wrapped_operand(node).map(|o| (o, false)), + AnnotatedExpression | LabeledExpression => node + .children() + .find(|child| { + super::is_operand(child) && !matches!(child.kind(), "annotation" | "label") + }) + .map(|o| (o, false)), UnaryExpression => { let operand = node.child_by_field_name("argument")?; match node.child_by_field_name("operator")?.kind_id().into() { diff --git a/src/metrics/abc/python.rs b/src/metrics/abc/python.rs index 0b016979..64306016 100644 --- a/src/metrics/abc/python.rs +++ b/src/metrics/abc/python.rs @@ -54,7 +54,7 @@ use crate::*; // stops on it instead. fn python_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { match node.kind_id().into() { - Python::ParenthesizedExpression => wrapped_operand(node), + Python::ParenthesizedExpression | Python::Await => wrapped_operand(node), Python::NamedExpression => node.child_by_field_name("value"), _ => None, } diff --git a/src/metrics/abc/rust.rs b/src/metrics/abc/rust.rs index 4d60b66d..831cd0a6 100644 --- a/src/metrics/abc/rust.rs +++ b/src/metrics/abc/rust.rs @@ -27,11 +27,33 @@ fn rust_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { use Rust::*; match node.kind_id().into() { - ParenthesizedExpression => wrapped_operand(node).map(|o| (o, false)), - Block | Block2 => last_operand(node).map(|o| (o, false)), + AwaitExpression => { + // Only an awaited async block yields its body; a bare one is a future. + let mut operand = wrapped_operand(node)?; + while let Some(inner) = rust_transparent_operand(&operand) { + operand = inner; + } + if operand.kind_id() == AsyncBlock as u16 { + operand = wrapped_operand(&operand)?; + } + Some((operand, false)) + } UnaryExpression if node.child(0).is_some_and(|c| c.kind_id() == BANG as u16) => { wrapped_operand(node).map(|o| (o, true)) } + _ => rust_transparent_operand(node).map(|o| (o, false)), + } +} + +fn rust_transparent_operand<'a>(node: &Node<'a>) -> Option> { + use Rust::*; + + match node.kind_id().into() { + ParenthesizedExpression | UnsafeBlock => wrapped_operand(node), + ConstBlock => node.child_by_field_name("body"), + TypeCastExpression => node.child_by_field_name("value"), + Block | Block2 => last_operand(node), + ExpressionStatement if !node.is_child(SEMI as u16) => wrapped_operand(node), _ => None, } } diff --git a/src/metrics/abc/slot_pays_tests.rs b/src/metrics/abc/slot_pays_tests.rs index 1f605e0a..39f67992 100644 --- a/src/metrics/abc/slot_pays_tests.rs +++ b/src/metrics/abc/slot_pays_tests.rs @@ -185,6 +185,32 @@ fn python_elif_is_an_else_and_a_slot() { ); } +#[test] +#[cfg(feature = "python")] +fn python_await_preserves_condition_counts() { + let source = r"class AwaitableComparison: + def __gt__(self, other): + async def result(): + return True + return result() + +async def f(a, b): + if PRED: + pass +"; + assert_rows( + LANG::Python, + source, + &[ + ("await b", 1, 1), + ("await (b > 1)", 1, 1), + ("await ( # comment\n b > 1)", 1, 1), + ("not await (b > 1)", 1, 1), + ("a and await (b > 1)", 2, 2), + ], + ); +} + /// The rows every JS-family grammar shares: the issue's predicates, each /// beside the bare identifier and the comparison it must neither fall /// below nor double. @@ -257,6 +283,28 @@ fn assert_js_family_slots(lang: LANG) { &function.replace(SLOT, "g(PRED);"), &[("b", 0, 0), ("!b", 1, 0), ("!this", 1, 0)], ); + assert_rows( + lang, + "async function f(a, b) { if (PRED) {} }", + &[ + ("await b", 1, 1), + ("await (b > 1)", 1, 1), + ("await /* comment */ (b > 1)", 1, 1), + ("await (a && b)", 2, 2), + ("a && await (b > 1)", 2, 2), + ("!await (b > 1)", 1, 1), + ("await await (b > 1)", 1, 1), + ], + ); + assert_rows( + lang, + "async function f(b) { return PRED; }", + &[ + ("await b", 0, 0), + ("await !b", 1, 0), + ("!await (b > 1)", 1, 0), + ], + ); } #[test] @@ -333,6 +381,13 @@ const C_FAMILY_IF_ROWS: &[(&str, u64, u64)] = &[ ("(int)(b > 1)", 1, 1), ("(int)b", 1, 1), ("a, b > 1", 1, 1), + ("({ b; })", 1, 1), + ("({ b > 1; })", 1, 1), + ("({ a; /* before */ b > 1; /* after */ })", 1, 1), + ("({ b > 1; a; })", 2, 1), + ("({ a > 1 && b > 1; })", 2, 2), + ("a && ({ b > 1; })", 2, 2), + ("!({ b > 1; })", 1, 1), ("!(b > 1)", 1, 1), ("b > 1", 1, 1), ("!b", 1, 1), @@ -382,6 +437,8 @@ fn assert_c_family_slots(lang: LANG) { ("!*p", 1, 0), ("!\"s\"", 1, 0), ("!(b > 1)", 1, 0), + ("({ !b; })", 1, 0), + ("!({ b > 1; })", 1, 0), ], ); } @@ -485,6 +542,31 @@ fn assert_cpp_slots(lang: LANG) { "template bool f(T a) requires PRED { return a && a; }", &with_body, ); + assert_rows( + lang, + r"#include +struct Task { + struct promise_type { + Task get_return_object() { return {}; } + std::suspend_never initial_suspend() { return {}; } + std::suspend_never final_suspend() noexcept { return {}; } + void return_void() {} + void unhandled_exception() {} + }; +}; +struct Awaitable { + bool await_ready() const { return true; } + void await_suspend(std::coroutine_handle<>) const {} + bool await_resume() const { return true; } + Awaitable operator>(int) const { return {}; } +}; +Task f(Awaitable b) { if (PRED) {} co_return; }", + &[ + ("co_await b", 1, 1), + ("co_await (b > 1)", 1, 1), + ("co_await /* comment */ (b > 1)", 1, 1), + ], + ); } #[test] @@ -534,6 +616,16 @@ fn rust_slots_pay_for_any_predicate() { // A block evaluates to its last expression: the comparison // inside pays, the slot does not pay again. ("{ x > 1 }", 1, 1), + ("unsafe { x > 1 }", 1, 1), + ("unsafe { /* comment */ x > 1 }", 1, 1), + ("const { 2 > 1 }", 1, 1), + ("const { true }", 1, 1), + ("b as bool", 1, 1), + ("(x > 1) as bool", 1, 1), + ("(a && b) as bool", 2, 2), + ("unsafe { a && b }", 2, 2), + ("a && unsafe { x > 1 }", 2, 2), + ("unsafe { x > 1; a }", 2, 1), // The match arm is a decision of its own; the slot tests the // value the match yields. ("match x { 1 => true, _ => false }", 2, 2), @@ -571,7 +663,36 @@ fn rust_slots_pay_for_any_predicate() { assert_rows( LANG::Rust, &function.replace(SLOT, "g(PRED);"), - &[("*p", 0, 0), ("!*p", 1, 0), ("!(a & b)", 1, 0)], + &[ + ("*p", 0, 0), + ("!*p", 1, 0), + ("!(a & b)", 1, 0), + ("unsafe { !b }", 1, 0), + ("!unsafe { x > 1 }", 1, 0), + ], + ); + assert_rows( + LANG::Rust, + "async fn f(a: bool, x: i32) { if PRED {} }", + &[ + ("async { a }.await", 1, 1), + ("async { x > 1 }.await", 1, 1), + ("((async { x > 1 })).await", 1, 1), + ("{ async { x > 1 } }.await", 1, 1), + ("unsafe { async { x > 1 } }.await", 1, 1), + ("async move { x > 1 }.await", 1, 1), + ("async { a && x > 1 }.await", 2, 2), + ("!async { x > 1 }.await", 1, 1), + ], + ); + assert_rows( + LANG::Rust, + "async fn f(b: bool) { g(PRED); }", + &[ + ("async { !b }", 0, 0), + ("async { !b }.await", 1, 0), + ("!async { !b }.await", 1, 0), + ], ); } @@ -797,6 +918,62 @@ fn csharp_1475_constructs_pay_their_slot() { ); } +#[test] +#[cfg(feature = "csharp")] +fn csharp_checked_wrappers_preserve_condition_counts() { + for wrapper in ["checked", "unchecked"] { + let function = format!( + "class C {{ bool f(bool a, bool b, int x) {{ if ({wrapper}(PRED)) {{}} return false; }} }}" + ); + assert_rows( + LANG::Csharp, + &function, + &[ + ("b", 1, 1), + ("x > 1", 1, 1), + ("/* comment */ x > 1", 1, 1), + ("a && b", 2, 2), + ("!(x > 1)", 1, 1), + ("checked(unchecked(x > 1))", 1, 1), + ], + ); + assert_rows( + LANG::Csharp, + &format!("class C {{ bool f(bool b, int x) {{ return {wrapper}(PRED); }} }}"), + &[("b", 0, 0), ("!b", 1, 0), ("!(x > 1)", 1, 0)], + ); + } + assert_rows( + LANG::Csharp, + "class C { async Task f(bool a, Task b, Task c) \ + { if (PRED) {} return false; } }", + &[ + ("await b", 1, 1), + ("await (a ? b : c)", 2, 2), + ("await /* comment */ (a ? b : c)", 2, 2), + ], + ); +} + +#[test] +#[cfg(feature = "kotlin")] +fn kotlin_labeled_and_annotated_wrappers_preserve_condition_counts() { + for wrapper in ["pred@ (PRED)", "@Suppress(\"unused\") (PRED)"] { + let template = format!("fun f(a: Boolean, b: Boolean, x: Int) {{ if ({wrapper}) {{}} }}"); + assert_rows( + LANG::Kotlin, + &template, + &[ + ("b", 1, 1), + ("x > 1", 1, 1), + ("/* comment */ x > 1", 1, 1), + ("a && b", 2, 2), + ("!(x > 1)", 1, 1), + ], + ); + } +} + /// PHP is truthy-valued, so every predicate is a decision. `??` is no /// condition token in PHP, so a slot holding one pays for it. #[test] From 42f90bd386ce9616a8fb79898b804b0e7862c622 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:44:07 -0700 Subject: [PATCH 30/34] fix(cognitive): preserve Bash sequences through unary tests --- src/metrics/bash_test_connective_tests.rs | 19 +++++++++++++++++++ src/metrics/cognitive/bash.rs | 14 ++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/metrics/bash_test_connective_tests.rs b/src/metrics/bash_test_connective_tests.rs index fe1e3710..95b7eb0d 100644 --- a/src/metrics/bash_test_connective_tests.rs +++ b/src/metrics/bash_test_connective_tests.rs @@ -220,3 +220,22 @@ fn bash_comparison_runs_score_like_their_twins_1536() { ); } } + +#[cfg(feature = "bash")] +#[test] +fn bash_unary_test_preserves_the_comparison_sequence() { + for (operator, connective) in [("-a", "&&"), ("-o", "||")] { + for unary in [r#"-n "$b""#, r#"! -z "$b""#, r#"-f "$b""#] { + for comparison in ["=", "==", "!="] { + let single = format!( + r#"if [ {unary} {operator} "$a" {comparison} 1 {operator} "$c" {comparison} 2 ]; then :; fi"# + ); + let twin = format!( + r#"if [ {unary} ] {connective} [ "$a" {comparison} 1 ] {connective} [ "$c" {comparison} 2 ]; then :; fi"# + ); + assert_eq!(measure(&twin), [5, 4, 2], "{twin}"); + assert_eq!(measure(&single), [5, 4, 2], "{single}"); + } + } + } +} diff --git a/src/metrics/cognitive/bash.rs b/src/metrics/cognitive/bash.rs index 751501ed..be07e9c0 100644 --- a/src/metrics/cognitive/bash.rs +++ b/src/metrics/cognitive/bash.rs @@ -39,10 +39,9 @@ const RUN_COMPARISONS: [u16; 4] = [ ]; /// The comparison runs the walk is inside, innermost last. A run member -/// is a `BinaryExpression3` whose operator is one of -/// [`RUN_COMPARISONS`]; each records its span and the end of the -/// outermost member of its run, inherited when its parent is a member -/// too. +/// is a comparison or an intervening unary test; each records its span +/// and the end of the outermost member of its run, inherited when its +/// parent is a member too. /// /// The walk is pre-order, so a member is visited before everything under /// it, and one whose span does not hold the node being visited is @@ -65,7 +64,7 @@ struct RunMember { } impl ComparisonRuns { - /// The byte a boolean sequence under `node`, a `BinaryExpression3` + /// The byte a boolean sequence under a binary expression or unary test /// whose parent is `parent`, extends to: the end of the run `parent` /// is a member of, else `node`'s own end. Records `node` when it is a /// member itself. @@ -82,7 +81,7 @@ impl ComparisonRuns { (Some(parent), Some(member)) if member.id == parent.id() => member.run_end, _ => end, }; - if node.wraps_any(&RUN_COMPARISONS) { + if node.kind() == "unary_expression" || node.wraps_any(&RUN_COMPARISONS) { self.0.push(RunMember { id: node.id(), start, @@ -140,6 +139,9 @@ impl Cognitive for BashCode { List => { compute_booleans(node, stats, AMPAMP, PIPEPIPE); } + UnaryExpression | UnaryExpression2 | UnaryExpression3 => { + stats.bash_runs.sequence_end(node, ancestors.parent(node)); + } BinaryExpression3 => { let end = stats.bash_runs.sequence_end(node, ancestors.parent(node)); compute_booleans_by_node(node, end, stats, |child| match child.kind_id().into() { From fe7601405acb425b5f76d041c1a109881a1a3b5d Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:44:07 -0700 Subject: [PATCH 31/34] fix(hooks): validate Python config with project tooling --- .pre-commit-config.yaml | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 989acff3..705f743b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,8 @@ repos: - id: check-symlinks - id: check-yaml - id: check-json - exclude: enums/templates/json.json + # Pyright accepts JSONC comments; py-typecheck validates its configuration. + exclude: ^(enums/templates/json\.json|big-code-analysis-py/pyrightconfig\.json)$ - repo: https://github.com/marco-c/taskcluster_yml_validator rev: v0.0.12 hooks: @@ -559,31 +560,17 @@ repos: # because pre-commit's isolated venv cannot see the project's # `maturin develop`'d `_native` extension or the rest of the # dev environment — under that isolation, `--strict`'s whole- - # program analysis is silently meaningless. The system hook runs - # against the contributor's PATH-resolved `mypy`, same pattern - # as the clippy/udeps/test hooks above. - # - # pyright is intentionally NOT a pre-commit hook: it bundles a - # Node runtime that pre-commit cannot hermetically isolate and - # suffers the same isolated-venv problem mypy does. The CI - # `python-lint` job covers it. + # program analysis is silently meaningless. The Makefile resolves + # mypy and pyright from the bootstrapped project venv before PATH, + # matching the canonical gate without requiring manual activation. - id: py-mypy name: py-mypy language: system # Match both `.py` source and `.pyi` stubs — editing # `_native.pyi` must also re-run mypy so the stub stays in # sync with the Rust-side ABI it describes. - files: '^big-code-analysis-py/.*\.pyi?$' - # Use --config-file + absolute-ish paths so the invocation is - # cross-platform — `bash -c "cd X && ..."` fails on native - # Windows pre-commit (no bash on cmd/PowerShell PATH). - entry: mypy - args: - - --strict - - --config-file=big-code-analysis-py/pyproject.toml - - big-code-analysis-py/python - - big-code-analysis-py/tests - - big-code-analysis-py/examples + files: '^big-code-analysis-py/(.*\.pyi?|pyrightconfig\.json|pyproject\.toml)$' + entry: make py-typecheck pass_filenames: false default_language_version: From 521abff6df2167fe172c6793840f572ee9bfd1fe Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:04:45 -0700 Subject: [PATCH 32/34] fix(deps): resolve CI dependency audit failures Upgrade rustls to 0.23.46 and replace serde_yaml with serde-saphyr to remove unsafe-libyaml. Preserve YAML data and round-trip behavior with updated tests and explicit string/float coverage. --- CHANGELOG.md | 7 ++ Cargo.lock | 94 ++++++++++++++----- Cargo.toml | 6 +- big-code-analysis-cli/Cargo.toml | 2 +- big-code-analysis-cli/src/formats.rs | 44 +++++++-- .../tests/output/format_smoke.rs | 8 +- .../tests/output/output_unification.rs | 2 +- big-code-analysis-cli/tests/vcs/vcs_trend.rs | 8 +- big-code-analysis-web/Cargo.toml | 2 +- big-code-analysis-web/src/web/negotiate.rs | 6 +- big-code-analysis-web/src/web/server_tests.rs | 18 ++-- src/wire.rs | 27 +++--- src/wire_ops_tests.rs | 4 +- 13 files changed, 155 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8645d26..e1752f47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,13 @@ for historical reference. ### Fixed +- **Dependency-audit failures in CI:** update the locked `rustls` to + 0.23.46 for RUSTSEC-2026-0346 through RUSTSEC-2026-0349, and replace + `serde_yaml` with `serde-saphyr` in CLI / REST YAML output and the + library's round-trip tests. This removes the unmaintained + `unsafe-libyaml` dependency (RUSTSEC-2024-0449). YAML keeps its data + schema; scalar quoting and layout may differ between serializers. + - **Bash cognitive complexity keeps a unary-test prefix in its logical sequence.** `[ -n "$b" -a "$a" = 1 -a "$c" = 2 ]` now scores like separate tests joined with `&&`; the `-o` / `||` spellings agree too. diff --git a/Cargo.lock b/Cargo.lock index 39437187..28e5875e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,6 +250,17 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "annotate-snippets" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f211a51805bc641f3ad5b7664c77d2547af685cc33b4cd8d31964027a46f13f1" +dependencies = [ + "anstyle", + "memchr", + "unicode-width", +] + [[package]] name = "anstream" version = "1.0.0" @@ -309,6 +320,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayvec" version = "0.7.8" @@ -430,8 +447,8 @@ dependencies = [ "quick-xml", "regex", "serde", + "serde-saphyr", "serde_json", - "serde_yaml", "sha2", "tempfile", "termcolor", @@ -507,8 +524,8 @@ dependencies = [ "predicates", "quick-xml", "serde", + "serde-saphyr", "serde_json", - "serde_yaml", "tempfile", "toml", ] @@ -541,8 +558,8 @@ dependencies = [ "predicates", "pretty_assertions", "serde", + "serde-saphyr", "serde_json", - "serde_yaml", "tempfile", "tokio", "tracing", @@ -1225,6 +1242,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "encoding_rs_io" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba3fe847045ecff794b9c138293a80db914678c453ad63fbf0c6a9eb6e00b22" +dependencies = [ + "encoding_rs", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -2311,6 +2337,16 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "granit-parser" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da55feca5d1625466bae068f9fd4b87892caeae70c3fe671ddb0f50d41aebf37" +dependencies = [ + "arraydeque", + "smallvec", +] + [[package]] name = "h2" version = "0.4.18" @@ -3017,6 +3053,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e94e1e6445d314f972ff7395df2de295fe51b71821694f0b0e1e79c4f12c8577" +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "nonempty" version = "0.12.0" @@ -3682,9 +3724,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "once_cell", @@ -3829,6 +3871,23 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-saphyr" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a253edfdea34032da21af84e16fcc94f4c7a6ba237f9f3bc2dbf43cea8535c44" +dependencies = [ + "annotate-snippets", + "base64", + "encoding_rs_io", + "granit-parser", + "nohash-hasher", + "num-traits", + "serde_core", + "smallvec", + "zmij", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -3884,19 +3943,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "sha1" version = "0.10.7" @@ -4718,16 +4764,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] -name = "unicode-xid" -version = "0.2.6" +name = "unicode-width" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] -name = "unsafe-libyaml" -version = "0.2.11" +name = "unicode-xid" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "untrusted" diff --git a/Cargo.toml b/Cargo.toml index 7c026657..aaee69c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,10 @@ authors = [ # entries via `.workspace = true`, so any grammar bump here MUST be # applied to enums/Cargo.toml in lockstep (see recreate-grammars.sh). [workspace.dependencies] +serde-saphyr = { version = "^2.0", default-features = false, features = [ + "serialize", + "deserialize", +] } tree-sitter = "=0.26.13" tree-sitter-bash = "=0.25.1" tree-sitter-c-sharp = "=0.23.5" @@ -378,7 +382,7 @@ globset = "^0.4" # three) to pin the uniform null/omit policy across all of them. Versions # mirror the `big-code-analysis-cli` pins; these crates already sit in the # workspace graph, so listing them here adds no new resolved dependency. -serde_yaml = "^0.9" +serde-saphyr.workspace = true ciborium = "^0.2" toml = "^1.1" diff --git a/big-code-analysis-cli/Cargo.toml b/big-code-analysis-cli/Cargo.toml index ff4a1fcf..a3cf7b4e 100644 --- a/big-code-analysis-cli/Cargo.toml +++ b/big-code-analysis-cli/Cargo.toml @@ -38,7 +38,7 @@ crossbeam = { version = "^0.8", features = ["crossbeam-channel"] } csv.workspace = true serde = { version = "^1.0", features = ["derive"] } serde_json = "^1.0" -serde_yaml = "^0.9" +serde-saphyr.workspace = true # `bca diff --since` materializes the tree at a git ref into an # auto-cleaning TempDir (#492), so tempfile is a runtime dependency, not # just a test helper. diff --git a/big-code-analysis-cli/src/formats.rs b/big-code-analysis-cli/src/formats.rs index 58929c98..b983e8a0 100644 --- a/big-code-analysis-cli/src/formats.rs +++ b/big-code-analysis-cli/src/formats.rs @@ -233,7 +233,7 @@ impl MetricsFormat { /// Bytes buffered in front of every output destination. /// /// The incremental serializers (`serde_json::to_writer`, -/// `serde_yaml::to_writer`, `ciborium::into_writer`) issue one write per +/// `serde_saphyr::to_io_writer`, `ciborium::into_writer`) issue one write per /// structural token, and both a raw `File` and a `StdoutLock` pass each /// one straight to the kernel: a 12 MB aggregate document measured /// 4.76 million `write(2)` calls, ~2.6 bytes apiece. 64 KiB is large @@ -339,7 +339,7 @@ pub(crate) fn dump_aggregate( output: &Path, pretty: bool, ) -> std::io::Result<()> { - write_buffered_file(output, |w| match format { + write_buffered_file(output, |mut w| match format { GenericFormat::Json => { if pretty { serde_json::to_writer_pretty(w, &items).map_err(ser_err) @@ -362,7 +362,7 @@ pub(crate) fn dump_aggregate( }; w.write_all(text.as_bytes()) } - GenericFormat::Yaml => serde_yaml::to_writer(w, &items).map_err(ser_err), + GenericFormat::Yaml => serde_saphyr::to_io_writer(&mut w, &items).map_err(ser_err), GenericFormat::Cbor => ciborium::into_writer(&items, w).map_err(ser_err), }) } @@ -645,7 +645,7 @@ struct Yaml; impl RenderDocument for Yaml { fn format(content: T) -> std::io::Result { - serde_yaml::to_string(&content).map_err(ser_err) + serde_saphyr::to_string(&content).map_err(ser_err) } } @@ -657,8 +657,8 @@ impl WriteFile for Yaml { path: &Path, output_path: &Path, ) -> std::io::Result<()> { - Self::with_file(path, output_path, |w| { - serde_yaml::to_writer(w, &content).map_err(ser_err) + Self::with_file(path, output_path, |mut w| { + serde_saphyr::to_io_writer(&mut w, &content).map_err(ser_err) }) } } @@ -736,7 +736,7 @@ pub(crate) fn write_json( /// Serialize `value` as YAML and write it through [`write_text`]. See /// [`write_json`] for why these live here. pub(crate) fn write_yaml(value: &T, output: Option<&PathBuf>) -> std::io::Result<()> { - let yaml = serde_yaml::to_string(value).map_err(std::io::Error::other)?; + let yaml = serde_saphyr::to_string(value).map_err(std::io::Error::other)?; write_text(&yaml, output) } @@ -1117,10 +1117,36 @@ mod tests { let yaml = dir.path().join("agg.yaml"); dump_aggregate(GenericFormat::Yaml, &expected, &yaml, false).expect("yaml"); let got: Vec = - serde_yaml::from_str(&std::fs::read_to_string(&yaml).expect("read")).expect("parse"); + serde_saphyr::from_str(&std::fs::read_to_string(&yaml).expect("read")).expect("parse"); assert_eq!(got, expected); } + #[test] + fn yaml_preserves_ambiguous_strings_and_full_precision() { + let expected = [ + "true", + "null", + "01", + "1e3", + "a: b", + "*alias", + "# comment", + "", + "café\nline\n", + ] + .map(|name| Item { + name: name.to_owned(), + value: 0.123_456_789_012_345_68, + }); + let text = Yaml::format(&expected).expect("render YAML"); + let parsed: Vec = serde_saphyr::from_str(&text).expect("parse YAML"); + assert_eq!(parsed, expected); + let directory = tempfile::tempdir().expect("tempdir"); + let path = directory.path().join("aggregate.yaml"); + dump_aggregate(GenericFormat::Yaml, &expected, &path, false).expect("write YAML"); + assert_eq!(std::fs::read_to_string(path).expect("read YAML"), text); + } + /// The `pretty` flag has to reach the serializer. Both spellings /// parse back to the same value, so a round-trip alone cannot tell /// them apart — the discriminating check is that exactly one of them @@ -1163,7 +1189,7 @@ mod tests { let yaml = dir.path().join("out.yaml"); write_yaml(&value, Some(&yaml)).expect("yaml"); assert_eq!( - serde_yaml::from_str::>(&std::fs::read_to_string(&yaml).expect("read")) + serde_saphyr::from_str::>(&std::fs::read_to_string(&yaml).expect("read")) .expect("parse"), value ); diff --git a/big-code-analysis-cli/tests/output/format_smoke.rs b/big-code-analysis-cli/tests/output/format_smoke.rs index 0b7e7e69..40e8bf82 100644 --- a/big-code-analysis-cli/tests/output/format_smoke.rs +++ b/big-code-analysis-cli/tests/output/format_smoke.rs @@ -461,11 +461,11 @@ fn cli_metrics_yaml_round_trips() { let fixture = write_rust_fixture(&dir); let out = run_metrics(&dir, "yaml", &fixture); - let doc: serde_yaml::Value = serde_yaml::from_str(&out).expect("metrics YAML parses"); - let map = doc.as_mapping().expect("YAML root is a mapping"); + let doc: serde_json::Value = serde_saphyr::from_str(&out).expect("metrics YAML parses"); + let map = doc.as_object().expect("YAML root is a mapping"); for key in STRUCTURAL_KEYS { assert!( - map.contains_key(serde_yaml::Value::String((*key).to_string())), + map.contains_key(*key), "YAML output missing top-level key {key:?}; doc was:\n{out}", ); } @@ -555,7 +555,7 @@ fn cli_metrics_all_formats_agree_on_metric() { let json = json_metric(&dir, &fixture, CYCLOMATIC_KEY); let yaml_out = run_metrics(&dir, "yaml", &fixture); - let yaml: serde_yaml::Value = serde_yaml::from_str(&yaml_out).expect("YAML parses"); + let yaml: serde_json::Value = serde_saphyr::from_str(&yaml_out).expect("YAML parses"); let yaml = yaml["metrics"][CYCLOMATIC_KEY.0][CYCLOMATIC_KEY.1] .as_f64() .expect("YAML metric is numeric"); diff --git a/big-code-analysis-cli/tests/output/output_unification.rs b/big-code-analysis-cli/tests/output/output_unification.rs index 2434a955..241868bc 100644 --- a/big-code-analysis-cli/tests/output/output_unification.rs +++ b/big-code-analysis-cli/tests/output/output_unification.rs @@ -529,7 +529,7 @@ fn per_file_documents_survive_spanning_several_output_buffers() { v.to_string() } "yaml" => { - let v: serde_yaml::Value = serde_yaml::from_slice(&bytes) + let v: serde_json::Value = serde_saphyr::from_slice(&bytes) .unwrap_or_else(|e| panic!("{format}: truncated or invalid document: {e}")); serde_json::to_string(&v).unwrap() } diff --git a/big-code-analysis-cli/tests/vcs/vcs_trend.rs b/big-code-analysis-cli/tests/vcs/vcs_trend.rs index d1f3c505..502c91fe 100644 --- a/big-code-analysis-cli/tests/vcs/vcs_trend.rs +++ b/big-code-analysis-cli/tests/vcs/vcs_trend.rs @@ -152,17 +152,17 @@ fn yaml_format_emits_valid_document() { .output() .expect("run bca vcs trend yaml"); assert!(output.status.success()); - let doc: serde_yaml::Value = - serde_yaml::from_slice(&output.stdout).expect("trend output is valid YAML"); + let doc: serde_json::Value = + serde_saphyr::from_slice(&output.stdout).expect("trend output is valid YAML"); assert_eq!(doc["trend_schema_version"], 1); // Non-vacuous structural checks: the point grid and file map are // present with the requested shape. assert_eq!( - doc["as_of_points"].as_sequence().map(Vec::len), + doc["as_of_points"].as_array().map(Vec::len), Some(2), "two requested points appear in the YAML" ); - assert!(doc["files"].is_mapping(), "files is a path-keyed mapping"); + assert!(doc["files"].is_object(), "files is a path-keyed mapping"); } /// A history-tuning flag (`--as-of`) is accepted in the *subcommand* diff --git a/big-code-analysis-web/Cargo.toml b/big-code-analysis-web/Cargo.toml index 79272c23..0f786c10 100644 --- a/big-code-analysis-web/Cargo.toml +++ b/big-code-analysis-web/Cargo.toml @@ -46,7 +46,7 @@ serde_json = "^1.0" # same value. Versions mirror the `big-code-analysis-cli` pins; both # crates already sit in the workspace graph, so this adds no new resolved # dependency. -serde_yaml = "^0.9" +serde-saphyr.workspace = true ciborium = "^0.2" tokio = { version = "1", features = ["rt", "sync", "time"] } tracing = "^0.1" diff --git a/big-code-analysis-web/src/web/negotiate.rs b/big-code-analysis-web/src/web/negotiate.rs index bd551f34..7118ac3c 100644 --- a/big-code-analysis-web/src/web/negotiate.rs +++ b/big-code-analysis-web/src/web/negotiate.rs @@ -57,7 +57,7 @@ impl Format { pub(super) fn encode(self, value: &T) -> Result, ()> { match self { Format::Json => serde_json::to_vec(value).map_err(|_| ()), - Format::Yaml => serde_yaml::to_string(value) + Format::Yaml => serde_saphyr::to_string(value) .map(String::into_bytes) .map_err(|_| ()), Format::Cbor => { @@ -278,8 +278,8 @@ mod tests { assert_eq!(parsed["n"], serde_json::json!(7)); let yaml = Format::Yaml.encode(&value).unwrap(); - let parsed: serde_yaml::Value = serde_yaml::from_slice(&yaml).unwrap(); - assert_eq!(parsed["n"], serde_yaml::Value::from(7)); + let parsed: serde_json::Value = serde_saphyr::from_slice(&yaml).unwrap(); + assert_eq!(parsed["n"], serde_json::Value::from(7)); let cbor = Format::Cbor.encode(&value).unwrap(); let parsed: ciborium::Value = ciborium::from_reader(cbor.as_slice()).unwrap(); diff --git a/big-code-analysis-web/src/web/server_tests.rs b/big-code-analysis-web/src/web/server_tests.rs index 1bb9c1d3..094950ef 100644 --- a/big-code-analysis-web/src/web/server_tests.rs +++ b/big-code-analysis-web/src/web/server_tests.rs @@ -4237,11 +4237,11 @@ async fn test_web_metrics_yaml_accept_emits_yaml() { let body = test::read_body(resp).await; // The body must parse as YAML and carry the same fields the JSON body // would, proving the response is genuinely YAML and not JSON mislabelled. - let parsed: serde_yaml::Value = serde_yaml::from_slice(&body).unwrap(); - assert_eq!(parsed["id"], serde_yaml::Value::from("neg-657")); - assert_eq!(parsed["language"], serde_yaml::Value::from("python")); + let parsed: Value = serde_saphyr::from_slice(&body).unwrap(); + assert_eq!(parsed["id"], Value::from("neg-657")); + assert_eq!(parsed["language"], Value::from("python")); // YAML is a JSON superset, so a successful YAML parse alone does not rule - // out a JSON body mislabelled as YAML. serde_yaml emits this nested map + // out a JSON body mislabelled as YAML. serde_saphyr emits this nested map // in block style (`id: neg-657\n...`), which is not valid JSON — so a // failed JSON parse is positive proof the bytes are real YAML. assert!( @@ -4286,10 +4286,8 @@ async fn test_web_metrics_yaml_matches_json_payload() { let yaml_resp = test::call_service(&app, metrics_request(Some("application/yaml")).to_request()).await; let yaml_body = test::read_body(yaml_resp).await; - let yaml: serde_yaml::Value = serde_yaml::from_slice(&yaml_body).unwrap(); - // Re-serialize the YAML value to JSON and compare structurally. - let yaml_as_json: Value = serde_json::to_value(&yaml).unwrap(); - assert_eq!(json, yaml_as_json); + let yaml: Value = serde_saphyr::from_slice(&yaml_body).unwrap(); + assert_eq!(json, yaml); } #[actix_rt::test] @@ -4353,8 +4351,8 @@ async fn test_web_ast_yaml_accept_emits_yaml() { assert_eq!(resp.status(), StatusCode::OK); assert_eq!(content_type(&resp), "application/yaml"); let body = test::read_body(resp).await; - let parsed: serde_yaml::Value = serde_yaml::from_slice(&body).unwrap(); - assert_eq!(parsed["id"], serde_yaml::Value::from("ct-515")); + let parsed: Value = serde_saphyr::from_slice(&body).unwrap(); + assert_eq!(parsed["id"], Value::from("ct-515")); } #[actix_rt::test] diff --git a/src/wire.rs b/src/wire.rs index 5d69764c..e7f86667 100644 --- a/src/wire.rs +++ b/src/wire.rs @@ -208,8 +208,8 @@ mod trend_wire_tests { assert!((from_json.ownership_top_share - row.ownership_top_share).abs() < 1e-12); // YAML round-trip. - let yaml = serde_yaml::to_string(&row).expect("serialize Vcs to YAML"); - let from_yaml: Vcs = serde_yaml::from_str(&yaml).expect("parse Vcs from YAML"); + let yaml = serde_saphyr::to_string(&row).expect("serialize Vcs to YAML"); + let from_yaml: Vcs = serde_saphyr::from_str(&yaml).expect("parse Vcs from YAML"); assert!(from_yaml.risk_score.is_nan() && from_yaml.burst.is_nan()); // CBOR round-trip. @@ -230,8 +230,9 @@ mod trend_wire_tests { vcs: sample_vcs(), }; - let yaml = serde_yaml::to_string(&point).expect("serialize VcsTrendPoint to YAML"); - let from_yaml: VcsTrendPoint = serde_yaml::from_str(&yaml).expect("parse point from YAML"); + let yaml = serde_saphyr::to_string(&point).expect("serialize VcsTrendPoint to YAML"); + let from_yaml: VcsTrendPoint = + serde_saphyr::from_str(&yaml).expect("parse point from YAML"); assert_eq!(from_yaml, point); let mut bytes = Vec::new(); @@ -264,8 +265,8 @@ mod trend_wire_tests { deltas: VcsTrendDeltas::default(), }; - let yaml = serde_yaml::to_string(&trend).expect("serialize VcsTrend to YAML"); - let from_yaml: VcsTrend = serde_yaml::from_str(&yaml).expect("parse VcsTrend from YAML"); + let yaml = serde_saphyr::to_string(&trend).expect("serialize VcsTrend to YAML"); + let from_yaml: VcsTrend = serde_saphyr::from_str(&yaml).expect("parse VcsTrend from YAML"); assert_eq!(from_yaml, trend); let mut bytes = Vec::new(); @@ -773,10 +774,10 @@ fn run() { #[test] fn yaml_round_trips() { check_func_space::(FIXTURE, "fixture.rs", |fs| { - let yaml = serde_yaml::to_string(&fs).expect("serialize to YAML"); - let back: FuncSpace = serde_yaml::from_str(&yaml).expect("parse wire from YAML"); + let yaml = serde_saphyr::to_string(&fs).expect("serialize to YAML"); + let back: FuncSpace = serde_saphyr::from_str(&yaml).expect("parse wire from YAML"); assert_eq!(back, fs.to_wire()); - assert_eq!(serde_yaml::to_string(&back).expect("re-serialize"), yaml); + assert_eq!(serde_saphyr::to_string(&back).expect("re-serialize"), yaml); }); } @@ -872,13 +873,13 @@ fn run() { "JSON null must deserialize back to NaN", ); - let yaml = serde_yaml::to_string(&mi).expect("YAML"); + let yaml = serde_saphyr::to_string(&mi).expect("YAML"); assert!( yaml.contains("original: null"), "non-finite must serialize to YAML null, got {yaml}", ); assert!( - serde_yaml::from_str::(&yaml) + serde_saphyr::from_str::(&yaml) .expect("parse") .original .is_nan() @@ -1118,7 +1119,7 @@ fn run() { ( [ serde_json::to_string(&at_limit).is_ok(), - serde_yaml::to_string(&at_limit).is_ok(), + serde_saphyr::to_string(&at_limit).is_ok(), toml::to_string(&at_limit).is_ok(), { let mut bytes = Vec::new(); @@ -1127,7 +1128,7 @@ fn run() { ], [ serde_json::to_string(&past_limit).is_ok(), - serde_yaml::to_string(&past_limit).is_ok(), + serde_saphyr::to_string(&past_limit).is_ok(), toml::to_string(&past_limit).is_ok(), { let mut bytes = Vec::new(); diff --git a/src/wire_ops_tests.rs b/src/wire_ops_tests.rs index af5a337d..3ea2e06c 100644 --- a/src/wire_ops_tests.rs +++ b/src/wire_ops_tests.rs @@ -87,8 +87,8 @@ fn borrowed_and_owned_ops_projections_serialize_alike() { "{lang:?} JSON must not depend on the projection" ); assert_eq!( - serde_yaml::to_string(&ops).expect("borrowed YAML"), - serde_yaml::to_string(&owned).expect("owned YAML"), + serde_saphyr::to_string(&ops).expect("borrowed YAML"), + serde_saphyr::to_string(&owned).expect("owned YAML"), "{lang:?} YAML must not depend on the projection" ); assert_eq!( From dcfdf2bf4fb96b8c16a4ca433d7182a3e0e1c7e5 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:33:27 -0700 Subject: [PATCH 33/34] refactor(elixir): reuse call context and cover lookup boundaries Remove obsolete timeout-clause lookup and carry the recognized argument list into ABC. Add boundary tests for short source buffers, root nodes, empty arguments and standalone keyword lists, preserving metric values while reducing uncovered production paths. --- CHANGELOG.md | 7 ++ .../src/lang_helpers/elixir.rs | 106 ++++++++++++++++-- src/metrics/abc/elixir.rs | 11 +- src/metrics/cognitive/elixir.rs | 11 +- src/metrics/elixir_decisions.rs | 79 +++++++++---- src/metrics/elixir_spelling_parity_tests.rs | 1 + 6 files changed, 173 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1752f47..0838515d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,13 @@ for historical reference. ### Changed +- **Elixir helper cleanup:** remote short-circuit call recognition now + returns its argument list with the call and ancestry, avoiding a second + lookup in ABC. The obsolete per-clause timeout lookup is removed. + `elixir_applying_remote_call` returns `ElixirRemoteCall` in the internal + `big-code-analysis-ast` crate, which is outside the public shape contract + described in [STABILITY.md](STABILITY.md). Metric values are unchanged. + - **Halstead bills angle brackets that enclose types as one `<>` operator**, like a generic list's (#1567). Rust qualified paths (`::f`, `::f`), C# function pointer types diff --git a/big-code-analysis-ast/src/lang_helpers/elixir.rs b/big-code-analysis-ast/src/lang_helpers/elixir.rs index 0f3e211e..b6a36dc5 100644 --- a/big-code-analysis-ast/src/lang_helpers/elixir.rs +++ b/big-code-analysis-ast/src/lang_helpers/elixir.rs @@ -144,6 +144,10 @@ pub fn elixir_applying_operator<'a>( /// ids each (grammar-dispatch §1). #[must_use] pub fn elixir_kernel_short_circuit_name<'a>(call: &Node<'a>, code: &[u8]) -> Option> { + kernel_short_circuit_parts(call, code).map(|(name, _)| name) +} + +fn kernel_short_circuit_parts<'a>(call: &Node<'a>, code: &[u8]) -> Option<(Node<'a>, Node<'a>)> { if call.kind() != "call" { return None; } @@ -162,9 +166,9 @@ pub fn elixir_kernel_short_circuit_name<'a>(call: &Node<'a>, code: &[u8]) -> Opt } else { name }; - (elixir_short_circuit_name(&name, code).is_some() - && call.children().any(|child| child.kind() == "arguments")) - .then_some(name) + elixir_short_circuit_name(&name, code) + .and_then(|_| call.children().find(|child| child.kind() == "arguments")) + .map(|arguments| (name, arguments)) } /// The short-circuit operation an operator token kind applies, keyed by @@ -198,9 +202,19 @@ pub fn elixir_short_circuit_name(name: &Node, code: &[u8]) -> Option { } } +/// A recognized short-circuit `Kernel` call and the context its operands need. +pub struct ElixirRemoteCall<'tree, 'chain> { + /// The call that applies the operator. + pub call: Node<'tree>, + /// The argument list found while recognizing the call. + pub arguments: Node<'tree>, + /// The call's own ancestry. + pub ancestors: Ancestors<'tree, 'chain>, +} + /// The `Kernel` remote call that applies `op`, a short-circuit operator's /// name — the call [`elixir_kernel_short_circuit_name`] answers `op` for -/// — paired with that call's own ancestry, or `None` when `op` sits +/// — with its argument list and ancestry, or `None` when `op` sits /// anywhere else. A token's parents are the `operator_identifier`, then /// the `dot`, then the call; the `identifier` naming `Kernel.and` sits /// on the `dot` directly. Any other node fails on the first one or two @@ -211,7 +225,7 @@ pub fn elixir_applying_remote_call<'a, 'c>( op: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, 'c>, -) -> Option<(Node<'a>, Ancestors<'a, 'c>)> { +) -> Option> { let mut up = ancestors.iter(op); let (mut dot, _) = up.next()?; if dot.kind() == "operator_identifier" { @@ -221,7 +235,83 @@ pub fn elixir_applying_remote_call<'a, 'c>( return None; } let (call, call_ancestors) = up.next()?; - elixir_kernel_short_circuit_name(&call, code) - .is_some_and(|name| name.id() == op.id()) - .then_some((call, call_ancestors)) + kernel_short_circuit_parts(&call, code) + .filter(|(name, _)| name.id() == op.id()) + .map(|(_, arguments)| ElixirRemoteCall { + call, + arguments, + ancestors: call_ancestors, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ElixirCode; + use crate::node::Tree; + use crate::test_support::for_each_node_with_chain; + + #[cfg(feature = "elixir")] + #[test] + fn short_circuit_name_requires_the_identifiers_source_span() { + let source = b"Kernel.and(a, b)"; + let tree = Tree::new::(source); + let name = tree + .get_root() + .preorder() + .find(|node| { + node.kind_id() == Elixir::Identifier as u16 && node.utf8_text(source) == Some("and") + }) + .expect("fixture contains the operator name"); + assert_eq!( + elixir_short_circuit_name(&name, source), + Some(Elixir::AMPAMP) + ); + assert_eq!( + elixir_short_circuit_name(&name, &source[..name.end_byte() - 1]), + None + ); + } + + #[cfg(feature = "elixir")] + #[test] + fn remote_call_lookup_rejects_a_root_and_keeps_its_argument_list() { + let source = b"Kernel.and(a, b)\nKernel.||(c, d)\n"; + let mut recognized = Vec::new(); + for_each_node_with_chain::(source, |node, chain| { + let known = elixir_applying_remote_call(node, source, Ancestors::known(chain)); + let unknown = elixir_applying_remote_call(node, source, Ancestors::unknown()); + assert_eq!( + known.as_ref().map(|r| (r.call.id(), r.arguments.id())), + unknown.map(|r| (r.call.id(), r.arguments.id())) + ); + if chain.is_empty() { + assert!(known.is_none()); + } + if let Some(remote) = known { + assert_eq!(remote.arguments.kind(), "arguments"); + assert_eq!( + remote.ancestors.parent(&remote.call).map(|p| p.kind()), + Some("source") + ); + recognized.push(( + node.utf8_text(source) + .expect("operator is UTF-8") + .to_owned(), + remote + .arguments + .utf8_text(source) + .expect("arguments are UTF-8") + .to_owned(), + )); + } + }); + assert_eq!( + recognized, + [ + ("and".to_owned(), "(a, b)".to_owned()), + ("||".to_owned(), "(c, d)".to_owned()) + ] + ); + } } diff --git a/src/metrics/abc/elixir.rs b/src/metrics/abc/elixir.rs index 02ee25d8..94a72790 100644 --- a/src/metrics/abc/elixir.rs +++ b/src/metrics/abc/elixir.rs @@ -207,22 +207,21 @@ fn elixir_count_remote_chain<'a>( ancestors: Ancestors<'a, '_>, conditions: &mut f64, ) { - let Some((call, call_ancestors)) = elixir_applying_remote_call(op, code, ancestors) else { + let Some(remote) = elixir_applying_remote_call(op, code, ancestors) else { return; }; let score = |operand: &Node, conditions: &mut f64| { elixir_count_condition(operand, code, conditions); }; - if let Some(piped) = call_ancestors - .parent(&call) + if let Some(piped) = remote + .ancestors + .parent(&remote.call) .filter(|pipe| elixir_binary_operator_is(pipe, |op| op == Elixir::PIPEGT)) .and_then(|pipe| pipe.child_by_field_name("left")) { score(&piped, conditions); } - if let Some(arguments) = call.children().find(|child| child.kind() == "arguments") { - count_each_operand(&arguments, score, conditions); - } + count_each_operand(&remote.arguments, score, conditions); } // What an Elixir `Call` contributes. The classification is by keyword diff --git a/src/metrics/cognitive/elixir.rs b/src/metrics/cognitive/elixir.rs index bd3a1b6d..a2c89242 100644 --- a/src/metrics/cognitive/elixir.rs +++ b/src/metrics/cognitive/elixir.rs @@ -75,13 +75,14 @@ fn compute_elixir_remote_booleans<'a>( ancestors: Ancestors<'a, '_>, stats: &mut Stats, ) { - if let Some((call, _)) = elixir_applying_remote_call(op, code, ancestors) + if let Some(remote) = elixir_applying_remote_call(op, code, ancestors) && let Some(key) = elixir_short_circuit_name(op, code) { - stats.structural = - stats - .boolean_seq - .eval_based_on_prev(key as u16, call.end_byte(), stats.structural); + stats.structural = stats.boolean_seq.eval_based_on_prev( + key as u16, + remote.call.end_byte(), + stats.structural, + ); } } diff --git a/src/metrics/elixir_decisions.rs b/src/metrics/elixir_decisions.rs index 2630a91a..12aa00b3 100644 --- a/src/metrics/elixir_decisions.rs +++ b/src/metrics/elixir_decisions.rs @@ -669,40 +669,73 @@ const KEYWORDS: &str = "keywords"; const ARGUMENTS: &str = "arguments"; /// The `Call` whose `do` section holds `node` (a `stab_clause`), in -/// either spelling ([`elixir_clause_section`]). The caller asks the -/// Call's keyword (`cond`, `for`). +/// either spelling: the block form's `do_block`, or a trailing `do:` +/// keyword pair. The caller asks the Call's keyword (`cond`, `for`). fn elixir_do_section_call<'a>( node: &Node<'a>, code: &'a [u8], ancestors: Ancestors<'a, '_>, ) -> Option> { - elixir_clause_section(node, code, ancestors) - .filter(|(key, _)| *key == "do") - .map(|(_, call)| call) -} - -/// The section holding `node` (a `stab_clause`) and the Call it belongs -/// to: `("do", call)` under the block form's `do_block`, whose parent is -/// the Call, `("after", call)` under the `after_block` inside that -/// `do_block`, and the keyword form's `(key, call)` under a `do: (…)` / -/// `after: (…)` block, bare or bracketed (`block` → `pair` → the climb -/// [`elixir_pair_section`] makes). `None` for a clause anywhere else, -/// the block form's `else` / `rescue` / `catch` sections included, which -/// no caller asks the owner of. -fn elixir_clause_section<'a>( - node: &Node<'a>, - code: &'a [u8], - ancestors: Ancestors<'a, '_>, -) -> Option<(&'a str, Node<'a>)> { use Elixir as E; let mut up = ancestors.iter(node); let (holder, _) = up.next()?; let (next, next_ancestors) = up.next()?; match holder.kind_id().into() { - E::DoBlock => Some(("do", next)), - E::AfterBlock if next.kind_id() == E::DoBlock as u16 => Some(("after", up.next()?.0)), - E::Block => elixir_pair_section(&next, code, next_ancestors), + E::DoBlock => Some(next), + E::Block => elixir_pair_section(&next, code, next_ancestors) + .filter(|(key, _)| *key == "do") + .map(|(_, call)| call), _ => None, } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ElixirCode; + use crate::test_support::for_each_node_with_chain; + + #[cfg(feature = "elixir")] + #[test] + fn empty_call_arguments_have_no_trailing_keywords() { + let source = b"f()\ng(else: 1)\n"; + let mut found = Vec::new(); + for_each_node_with_chain::(source, |node, _| { + if node.kind() == ARGUMENTS { + found.push(elixir_trailing_keywords(node).map(|keywords| { + keywords + .utf8_text(source) + .expect("keywords are UTF-8") + .to_owned() + })); + } + }); + assert_eq!(found, [None, Some("else: 1".to_owned())]); + } + + #[cfg(feature = "elixir")] + #[test] + fn standalone_keyword_list_has_no_owning_call() { + let source = b"[else: 1]\nf(else: 2)\n"; + let mut found = Vec::new(); + for_each_node_with_chain::(source, |node, chain| { + if node.kind_id() == Elixir::Pair as u16 { + found.push( + elixir_pair_section(node, source, Ancestors::known(chain)).map( + |(key, call)| { + ( + key.to_owned(), + call.utf8_text(source).expect("call is UTF-8").to_owned(), + ) + }, + ), + ); + } + }); + assert_eq!( + found, + [None, Some(("else".to_owned(), "f(else: 2)".to_owned()))] + ); + } +} diff --git a/src/metrics/elixir_spelling_parity_tests.rs b/src/metrics/elixir_spelling_parity_tests.rs index 12c5a6b5..f2337b66 100644 --- a/src/metrics/elixir_spelling_parity_tests.rs +++ b/src/metrics/elixir_spelling_parity_tests.rs @@ -218,6 +218,7 @@ fn elixir_if_else_scores_alike_in_every_spelling() { #[cfg(feature = "elixir")] #[test] fn elixir_list_that_is_not_a_keyword_list_has_no_sections() { + assert_group("[1]", [1, 1, 0, 0], &["[else: 1]", "[do: 1, else: 2]"]); assert_group( "if(c, x)", [2, 2, 1, 1], From f3aedfc144427df409bcdd60ebb9c419c9914b73 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:39:19 -0700 Subject: [PATCH 34/34] ci(coverage): pin measurement tools and archive raw reports --- .github/workflows/ci.yml | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce3dd9e8..c66eed15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -320,21 +320,29 @@ jobs: with: fetch-depth: 1 submodules: recursive - # llvm-tools-preview supplies llvm-profdata/llvm-cov for source-based - # coverage instrumentation. + # Pin the measurement toolchain independently of the stable build jobs: + # compiler changes can move coverage regions without changing test coverage. - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # stable tip with: - toolchain: stable + toolchain: "1.99.1" components: llvm-tools-preview # Coverage builds carry -C instrument-coverage, so their artifacts are # incompatible with the regular build cache; a distinct key keeps the # two from thrashing each other. - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: - key: coverage + key: coverage-1.99.1-llvm-cov-0.9.1 - uses: taiki-e/install-action@9983c65e42da123ff25d1f78505eb6de315aa172 # v2.87.20 with: - tool: cargo-nextest,cargo-llvm-cov + tool: cargo-nextest@0.9.146,cargo-llvm-cov@0.9.1 + - name: Record coverage provenance + run: | + { + git rev-parse HEAD + rustc --version --verbose + cargo llvm-cov --version + cargo nextest --version + } > coverage-provenance.txt # nextest is the same runner the `test` job uses, so instrumenting it # keeps the coverage numbers aligned with what CI actually exercises. # --no-report defers reporting so the same profile data can be emitted @@ -344,6 +352,8 @@ jobs: # nightly, and pairing nightly with the workflow-wide `-D warnings` # would red-X CI on unrelated new nightly lints. The `test` job still # runs doctests for correctness — only their coverage is omitted. + - name: Clean coverage data + run: cargo llvm-cov clean --workspace - name: Run tests with coverage instrumentation run: cargo llvm-cov --no-report nextest --all-features --workspace --locked # Codecov's native format preserves region coverage (lcov would drop to @@ -358,6 +368,19 @@ jobs: # 103 files that are in fact ~98% covered. Keep the two in step. - name: Generate Codecov report run: cargo llvm-cov report --workspace --codecov --output-path codecov.json + - name: Generate raw LLVM coverage report + run: cargo llvm-cov report --workspace --json --output-path coverage-llvm.json + - name: Archive coverage reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-${{ github.sha }} + path: | + codecov.json + coverage-llvm.json + coverage-provenance.txt + if-no-files-found: error + retention-days: 30 # Per-file coverage table in the run's Summary tab, so reviewers can # read coverage without leaving GitHub or waiting on the Codecov PR # comment.