diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 2980efe2d..410b29329 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -279,7 +279,7 @@ value = 42.0 path = "big-code-analysis-ast/src/parser.rs" qualified = "Parser::filters" metric = "halstead.effort" -value = 87390.78947368421 +value = 90363.0 [[entry]] path = "big-code-analysis-ast/src/preproc.rs" @@ -1065,7 +1065,7 @@ value = 6.0 path = "src/ops.rs" qualified = "ops_inner" metric = "halstead.effort" -value = 70903.25755192844 +value = 84452.11438842113 [[entry]] path = "src/output/checkstyle.rs" @@ -1179,7 +1179,7 @@ value = 5.0 path = "src/spaces/compute.rs" qualified = "compute_per_node" metric = "halstead.effort" -value = 63645.7536117776 +value = 64334.51855807396 [[entry]] path = "src/spaces/compute.rs" @@ -1191,7 +1191,7 @@ value = 7.0 path = "src/spaces/compute.rs" qualified = "metrics_inner" metric = "halstead.effort" -value = 121169.85621346734 +value = 141885.98828207294 [[entry]] path = "src/spaces/compute.rs" diff --git a/.claude/rules/testing.md b/.claude/rules/testing.md index bb317cb98..6ac196dcc 100644 --- a/.claude/rules/testing.md +++ b/.claude/rules/testing.md @@ -651,6 +651,9 @@ tree at `ref` as well (`git archive` into a scratch directory), computes which single-language builds compile each test in each, and fails on a test that still exists but stopped being built somewhere. The probe set comes from the language table, so a new language extends it for free. +A test's membership includes the `cfg` on the `mod` line that loads its +file, so narrowing a root `mod` gate in a `tests/*/main.rs` is reported +too; before that, #1567's drop of `kotlin` from one went unseen. CI runs it per pull request against the base branch head — the PR's `base.sha`, not `git merge-base`, because the job checks out shallow and diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce3dd9e82..c66eed154 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -320,21 +320,29 @@ jobs: with: fetch-depth: 1 submodules: recursive - # llvm-tools-preview supplies llvm-profdata/llvm-cov for source-based - # coverage instrumentation. + # Pin the measurement toolchain independently of the stable build jobs: + # compiler changes can move coverage regions without changing test coverage. - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # stable tip with: - toolchain: stable + toolchain: "1.99.1" components: llvm-tools-preview # Coverage builds carry -C instrument-coverage, so their artifacts are # incompatible with the regular build cache; a distinct key keeps the # two from thrashing each other. - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: - key: coverage + key: coverage-1.99.1-llvm-cov-0.9.1 - uses: taiki-e/install-action@9983c65e42da123ff25d1f78505eb6de315aa172 # v2.87.20 with: - tool: cargo-nextest,cargo-llvm-cov + tool: cargo-nextest@0.9.146,cargo-llvm-cov@0.9.1 + - name: Record coverage provenance + run: | + { + git rev-parse HEAD + rustc --version --verbose + cargo llvm-cov --version + cargo nextest --version + } > coverage-provenance.txt # nextest is the same runner the `test` job uses, so instrumenting it # keeps the coverage numbers aligned with what CI actually exercises. # --no-report defers reporting so the same profile data can be emitted @@ -344,6 +352,8 @@ jobs: # nightly, and pairing nightly with the workflow-wide `-D warnings` # would red-X CI on unrelated new nightly lints. The `test` job still # runs doctests for correctness — only their coverage is omitted. + - name: Clean coverage data + run: cargo llvm-cov clean --workspace - name: Run tests with coverage instrumentation run: cargo llvm-cov --no-report nextest --all-features --workspace --locked # Codecov's native format preserves region coverage (lcov would drop to @@ -358,6 +368,19 @@ jobs: # 103 files that are in fact ~98% covered. Keep the two in step. - name: Generate Codecov report run: cargo llvm-cov report --workspace --codecov --output-path codecov.json + - name: Generate raw LLVM coverage report + run: cargo llvm-cov report --workspace --json --output-path coverage-llvm.json + - name: Archive coverage reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-${{ github.sha }} + path: | + codecov.json + coverage-llvm.json + coverage-provenance.txt + if-no-files-found: error + retention-days: 30 # Per-file coverage table in the run's Summary tab, so reviewers can # read coverage without leaving GitHub or waiting on the Codecov PR # comment. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 989acff33..705f743b0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,8 @@ repos: - id: check-symlinks - id: check-yaml - id: check-json - exclude: enums/templates/json.json + # Pyright accepts JSONC comments; py-typecheck validates its configuration. + exclude: ^(enums/templates/json\.json|big-code-analysis-py/pyrightconfig\.json)$ - repo: https://github.com/marco-c/taskcluster_yml_validator rev: v0.0.12 hooks: @@ -559,31 +560,17 @@ repos: # because pre-commit's isolated venv cannot see the project's # `maturin develop`'d `_native` extension or the rest of the # dev environment — under that isolation, `--strict`'s whole- - # program analysis is silently meaningless. The system hook runs - # against the contributor's PATH-resolved `mypy`, same pattern - # as the clippy/udeps/test hooks above. - # - # pyright is intentionally NOT a pre-commit hook: it bundles a - # Node runtime that pre-commit cannot hermetically isolate and - # suffers the same isolated-venv problem mypy does. The CI - # `python-lint` job covers it. + # program analysis is silently meaningless. The Makefile resolves + # mypy and pyright from the bootstrapped project venv before PATH, + # matching the canonical gate without requiring manual activation. - id: py-mypy name: py-mypy language: system # Match both `.py` source and `.pyi` stubs — editing # `_native.pyi` must also re-run mypy so the stub stays in # sync with the Rust-side ABI it describes. - files: '^big-code-analysis-py/.*\.pyi?$' - # Use --config-file + absolute-ish paths so the invocation is - # cross-platform — `bash -c "cd X && ..."` fails on native - # Windows pre-commit (no bash on cmd/PowerShell PATH). - entry: mypy - args: - - --strict - - --config-file=big-code-analysis-py/pyproject.toml - - big-code-analysis-py/python - - big-code-analysis-py/tests - - big-code-analysis-py/examples + files: '^big-code-analysis-py/(.*\.pyi?|pyrightconfig\.json|pyproject\.toml)$' + entry: make py-typecheck pass_filenames: false default_language_version: diff --git a/AGENTS.md b/AGENTS.md index ff2204ea6..e7e268620 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -681,7 +681,9 @@ stranded `.snap.new` drift that blocks CI on every subsequent change. The limits `bca init` scaffolds are defined once, in `big-code-analysis-cli/src/default_thresholds.rs`. They are derived from published thresholds plus a 20-language corpus measurement (#1140), -and each carries its rationale inline. Change them there, not in a +and each carries its rationale inline. `halstead.effort` was re-measured +in #1569 on a rebuilt corpus recorded in +[`docs/development/threshold_corpus.md`](docs/development/threshold_corpus.md). Change them there, not in a copy: the scaffolded `bca.toml` renders its `[thresholds]` block from that table, so there is no second copy of the numbers to drift. @@ -709,8 +711,8 @@ Three things to know before quoting a number at anyone: - **These are language-agnostic defaults, and no language is agnostic.** Measured 97.5th-percentile `cognitive` runs from 4 (C#) to 50 (C). C, Tcl, Bash, Lua, Perl, and Go want roughly double the - defaults; Java, Ruby, Kotlin, C#, and Elixir sit far under them. The - per-language table lives in + `cognitive` default; Java, Ruby, Kotlin, C#, and Elixir sit far + under it. The per-language table lives in [Choosing thresholds](big-code-analysis-book/src/recipes/thresholds.md). - **The right limit depends on the job.** A blocking CI gate, an agent edit loop, a legacy triage pass, and a safety-critical build want diff --git a/CHANGELOG.md b/CHANGELOG.md index c00dafa08..0838515d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,242 @@ for historical reference. ## [Unreleased] +### Added + +- **Thirteen Perl, Bash, Kotlin and Elixir scaling probes in the + complexity-class gate** (`make bench-scaling`). #1575 adds seven: the + Perl ABC misparse climb, Perl ABC's `=~` binding check under an + `ERROR` root, and the Bash cognitive / ABC comparison-run walks + (`abc/perl-or-chain`, `abc/perl-misparsed-comparison`, + `abc/perl-and-statements`, `abc/perl-error-tail`, + `cognitive/bash-test-connective-run`, `abc/bash-arith-assign-chain`, + `abc/bash-eq-and-test`). The rest price sibling lookups: + `abc/perl-bareword-keys` (#1572), `halstead/perl-error-tail-keys` + and `halstead/kotlin-short-interpolations` (#1584), + `count/perl-error-tail-calls` (#1580) and + `nexits/perl-error-tail-keys` (#1585). Each was watched tripping with + its pre-fix walk restored. `cognitive/elixir-mixed-arguments` prices + the keyword-section lookup #1574 added, whose first form read a call's + whole argument list once per keyword pair and once per keyword-form + clause: 5.4 s for a 16 000-argument call in a release build, now + linear. + +### Changed + +- **Elixir helper cleanup:** remote short-circuit call recognition now + returns its argument list with the call and ancestry, avoiding a second + lookup in ABC. The obsolete per-clause timeout lookup is removed. + `elixir_applying_remote_call` returns `ElixirRemoteCall` in the internal + `big-code-analysis-ast` crate, which is outside the public shape contract + described in [STABILITY.md](STABILITY.md). Metric values are unchanged. + +- **Halstead bills angle brackets that enclose types as one `<>` + operator**, like a generic list's (#1567). Rust qualified paths + (`::f`, `::f`), C# function pointer types + (`delegate*`) and Kotlin's qualified `super` each + billed the comparison `<` and `>` before. `N1` falls by one per + occurrence; `n1` loses any `<` or `>` that no comparison still uses, + and gains `<>` unless `<>` is already present. Kotlin's `id(a)` + (#1394) and the Rust macro token tree stay exceptions. +- **Java class literals bill their parts instead of one operand spelled + by the whole literal** (#1566). JLS §15.8.2 defines `Foo.class` as an + expression, not a literal, so `java.util.List.class` bills three `.` + operators plus the operands `java`, `util`, `List` and `class`; + `Foo[].class` bills `[]`, `.`, `Foo` and `class`; `int.class` bills + `int`, `.` and `class`. Groovy's `Foo.class` and Kotlin's and PHP's + `Foo::class` already scored this way. Before, the literal's `.`, `[]` + and primitive also billed beside the whole-text operand, so each + literal paid for part of its text twice. +- **`bca ops` lists Java's `boolean` as `boolean` instead of its node + kind `boolean_type`** (#1568), so Java and Groovy report the same + vocabulary for identical source. Java's Halstead counts are + unchanged. +- **The `halstead.effort` default was re-measured after the vocabulary + changes** (#1569), on a rebuilt 41-repository, 20-language corpus. It + stays at 50,000, flagging 2.2% of functions in the median language. + The book's per-language `halstead.effort` column is regenerated, and + the corpus is recorded with commit SHAs in + `docs/development/threshold_corpus.md` so it can be reproduced. + +### Fixed + +- **Dependency-audit failures in CI:** update the locked `rustls` to + 0.23.46 for RUSTSEC-2026-0346 through RUSTSEC-2026-0349, and replace + `serde_yaml` with `serde-saphyr` in CLI / REST YAML output and the + library's round-trip tests. This removes the unmaintained + `unsafe-libyaml` dependency (RUSTSEC-2024-0449). YAML keeps its data + schema; scalar quoting and layout may differ between serializers. + +- **Bash cognitive complexity keeps a unary-test prefix in its logical + sequence.** `[ -n "$b" -a "$a" = 1 -a "$c" = 2 ]` now scores like + separate tests joined with `&&`; the `-o` / `||` spellings agree too. +- **Perl dash-key recovery preserves package-qualified calls.** + `-foo::bar => 1` remains a negated call rather than an auto-quoted key, + retaining its ABC branch, `find` / `count` match and Halstead name. +- **C# generic-name recovery is linear in qualification depth** and + preserves real generic qualifiers beside recovered comparisons. + **Elixir receive timeout analysis is linear in clause count**, including + malformed input with multiple timeout clauses. +- **Language-test gates preserve call-site conditions** and distinguish + local variables from same-named helpers. Module-gate comparison also + follows nested `foo.rs` → `foo/bar.rs` declarations. The JSON hook now + leaves Pyright's JSONC configuration to the Python type-check gate. + +- **ABC no longer counts a predicate twice through value-preserving + wrappers.** This covers JavaScript, Mozjs, TypeScript and TSX `await`, + C# `checked` / `unchecked` and `await`, Python `await`, Rust unsafe / + const blocks, casts and awaited async blocks, Kotlin expression labels + and annotations, and GNU C-family statement expressions and C++ + `co_await`. Wrapped comparisons and logical chains keep their existing + condition counts; wrapped bare values still pay for their boolean slot. + +- **TypeScript and TSX bill their member modifiers as operators** + (#1563): `public`, `private`, `protected`, `override`, `readonly`, + `abstract` and `accessor`, as C# and Kotlin do, counting each keyword + once and never its `accessibility_modifier` / `override_modifier` + wrapper. JavaScript, Mozjs, TypeScript and TSX bill `static`; the + JavaScript and Mozjs one-token `static get` (a `static get` followed + by a newline) bills as one operator. The same words used as names + (`let readonly = 1`, `obj.static`) stay operands. +- **Kotlin bills `typealias`, `init`, `where`, `by`, `get` and `set` as + operators** (#1564), as C# bills `get`, `set`, `init` and `where`. + Each counts once, as the keyword and never as its wrapper node. The + `@get:` / `@set:` annotation targets count nothing, like `@field:`. + The words used as names (`val by = 1`, `x.get(i)`) and the accessor's + backing `field` stay operands. +- **Java and Groovy bill every modifier as an operator** (#1565), as C# + and Kotlin do. Before, only `abstract`, `final`, `transient`, + `synchronized` and `default` billed. Now `public`, `private`, + `protected`, `static`, `native`, `strictfp`, `volatile`, `sealed`, + `non-sealed` and the `permits` clause keyword bill as well, each as + its keyword and never as Java's `modifiers` wrapper. The `static` of + `import static` and `requires static` bills nothing, as `import` does + not, and the words used as names (`var sealed = 1`, Groovy's + `[static: 1]`) stay operands. +- **Groovy bills the primitive type names as operators keyed by their + spelling** (#1568): `boolean`, `byte`, `char`, `double`, `float`, + `int`, `long`, `short` and `void`, as Java does, covering declared + types, class-literal receivers (`int.class`) and uninitialised locals. + Where Groovy allows the reserved word as a name (`x.int`, `x?.int`, + `x::int`, `[int: 1]`, `f(int: 1)`) it stays an operand. +- **C# no longer bills an interpolated string's alignment `,` or format + `:`** (`$"{x,8:F2}"`) as operators (#1573), matching Python's + format-spec `:`. The alignment value still counts: `8` is an operand, + and `{x,-w}` counts `-` and `w`. A `,` or `:` inside the hole — a call + argument list, ternary or named argument — still counts. +- **C, C++ and Mozcpp bill every storage class, function specifier and + type qualifier as a Halstead operator** (#1578): `static`, `extern`, + `register`, `inline`, `thread_local`, `const`, `volatile`, `restrict`, + `_Atomic`, `constexpr`, `_Noreturn`, `alignas`, their vendor spellings + (`__inline`, `__forceinline`, `__restrict__`, `__thread`, + `__volatile__`, `__extension__`, …) and C's storage class `auto`. C++ + and Mozcpp also bill `virtual`, `mutable`, `constinit`, `consteval`, + `explicit`, `friend`, the access labels, `override`, `final` and the + `noexcept` of an exception specification. Each counts once as its + keyword, never as the `storage_class_specifier` / `type_qualifier` / + `virtual_specifier` / `access_specifier` wrapper. `override` and + `final` used as names stay operands, `extern "C"` keeps `"C"` an + operand, and C++'s placeholder `auto`, attributes, MS calling + conventions and the Gecko `MOZ_*` macros are unchanged. Halstead volume + rises and MI scores of C-family code fall. +- **Objective-C bills declaration modifiers as Halstead operators** + (#1579). C's storage classes, function specifiers and type qualifiers + bill as in C, so a `.m` scores a C declaration the same as a `.c` or + `.mm`. ObjC's own modifiers also bill: the `@public` / `@private` / + `@protected` / `@package` ivar labels, `@optional` / `@required`, the + ARC ownership qualifiers and `__bridge` casts, the nullability + qualifiers, `__kindof`, `__covariant` / `__contravariant`, `_Complex`, + the `in` / `out` / `inout` / `bycopy` / `byref` / `oneway` method-type + qualifiers, and property attribute words (`nonatomic`, `copy`, + `getter`, …). A `getter=` / `setter=` value stays an operand. Apple + SDK macros (`NS_INLINE`, `FOUNDATION_EXPORT`, `IBOutlet`, `__unused`) + bill nothing. +- **Perl bareword hash keys no longer score as calls** (#1572). + tree-sitter-perl 1.1.2 parses an auto-quoted bareword key as a call, + so `(name => 1)`, every named argument (`new(name => $x, size => 3)`) + and `{ a => { b => 1 } }` scored one ABC branch per key, and + `(-name => 1)` / `$h{-name}` also billed a Halstead `-` operator and + the unsigned operand `name`. Each key now scores like its quoted twin + and is billed as written (`name`, `-name`), including a word that + ends a longer expression before `=>` (`(1 - name => 2)`), since `=>` + quotes the word right before it. Package-qualified words + (`Foo::Bar => 1`), words with arguments (`name() => 1`), bareword + values (`(a => foo)`) and `$h{+shift}` keep their call. So does a + `-name` outside those two positions — a value (`(x => -name)`), a + comma-separated argument (`f(-text, 'hi')`) or a multi-key slice — + since there Perl's own reading depends on whether a sub of that name + is visible, which no single file says. Cyclomatic and cognitive are + unchanged. This is a local workaround for upstream #1571. +- **Perl `bca find --type call` and `bca count --type call` report each + call once and skip auto-quoted keys** (#1580). tree-sitter-perl parses + a call's name (`foo` in `foo(1)`, `print` in `print "x"`) as a bareword + call nested inside the call, and parses `(name => 1)`, `(-size => 2)` + and `$h{-name}` as calls too; the issue's fixture went from five + matches to one. The filter now applies the gate ABC already used, so a + Perl file's `call` count equals its `abc.branches`. A bareword with no + wrapper still counts: `foo;`, `&foo;`, and the wrapper-less `foo(1` + that error recovery leaves. New + `Checker::is_call_with_code(node, code, ancestors)` in + `big-code-analysis-ast`, forwarding to `is_call` by default. +- **Perl `nexits` no longer counts an auto-quoted `die` or `exit` hash + key as an exit point** (#1585): `(exit => 1, die => 2)` now scores as + its quoted twin does. A real `die` / `exit` call, including + `$h{die()}`, `die() => 1`, `(x => die)` and `CORE::die => 1`, still + counts. The crate-private `Exit::compute` now takes the ancestor + chain; no public API changes. +- **Elixir scores every spelling of a construct alike** (#1574). A + trailing bracketed keyword list (`if(c, [do: 1, else: 2])`, + `for(x <- xs, [reduce: 0, do: …])`) and a quoted key (`"else": 2`) now + score exactly as the bare keyword form and the `do … end` block do. + Before, the bracketed `with` lost its modified-cyclomatic `else` tier, + the bracketed `for … reduce:` scored its accumulator head as a + decision, the bracketed `cond` lost its free `true ->`, and the + bracketed `if` / `unless` lost ABC's `else` condition. + `Kernel.&&(a, b)` / `Kernel.||(a, b)`, including the piped + `a |> Kernel.||(b)`, now score ABC conditions and a cognitive boolean + sequence as `a && b` / `a || b` do; cyclomatic already counted them, + and a user module's `Foo.||(a, b)` still scores nothing. Cognitive now + scores the keyword forms' `else:` (+1) and `rescue:` / `catch:` + (+nesting) for `if`, `unless`, `try` and the `def` family, as their + blocks do; the keyword `try` scored 0 before. For any other owner the + keyword form still scores nothing — `else:` is an ordinary option to + a function — while its `do … end` block scores as before, so + `foo(do: 1, else: 2)` and `foo do 1 else 2 end` still differ by one + there. A `with`'s `else` adds + no cognitive cost in either spelling: its clauses are the arms of a + `case`, already paid by the container (the block form scored 2 + against the keyword form's 1). In `big-code-analysis-ast`, + `elixir_remote_call_applies` is replaced by + `elixir_applying_remote_call`, which returns the applying call and + its ancestor chain; `elixir_kernel_operator_token` is now + `elixir_kernel_short_circuit_name`. +- **Elixir `receive` scores like `case`** (#1581): modified cyclomatic + counts the construct once and cognitive complexity adds +nesting, in + the block and keyword forms. A `receive` gets one free arm, as a + `case` does: its bare `_ ->` if it has one, otherwise its `after` + timeout clause, which then no longer scores in standard cyclomatic or + ABC. A `receive` holding only an `after` scores nothing. +- **Elixir `Kernel.and(a, b)` and `Kernel.or(a, b)` count as + short-circuit decisions** in cyclomatic, ABC conditions and cognitive + complexity (#1582), matching `a and b` / `a or b` and the `Kernel.&&` + / `Kernel.||` calls. + +### Performance + +- **Perl ABC and Halstead no longer go quadratic on a file whose tail + fails to parse into a flat `ERROR` node** (#1584). The bareword-key, + dash-key, key-sign and misparse-operator lookups now read the children + the walk recorded (a `SiblingCache` lent through `Ancestors`) instead + of seeking from the `ERROR`'s start: 16 000 copies of `{ last,` take + 0.03 s, down from 2.2 s. The same change makes Kotlin Halstead linear + on a string of many short `$name` interpolations, which was quadratic + on valid input (8 000 names: 11.5 s → 0.01 s). In + `big-code-analysis-ast`, which `STABILITY.md` leaves outside the shape + contract, `AncestorIter::Chain` carries the lent cache as a second + field, `perl_dash_signs_keyword_key` takes the `-`'s ancestor chain in + place of its parent node, and `perl_is_dash_key_word` is + crate-private. + ## [2.3.1] - 2026-10-10 For the `big-code-analysis` crate this release changes metric values diff --git a/Cargo.lock b/Cargo.lock index 394371871..28e5875e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,6 +250,17 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "annotate-snippets" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f211a51805bc641f3ad5b7664c77d2547af685cc33b4cd8d31964027a46f13f1" +dependencies = [ + "anstyle", + "memchr", + "unicode-width", +] + [[package]] name = "anstream" version = "1.0.0" @@ -309,6 +320,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayvec" version = "0.7.8" @@ -430,8 +447,8 @@ dependencies = [ "quick-xml", "regex", "serde", + "serde-saphyr", "serde_json", - "serde_yaml", "sha2", "tempfile", "termcolor", @@ -507,8 +524,8 @@ dependencies = [ "predicates", "quick-xml", "serde", + "serde-saphyr", "serde_json", - "serde_yaml", "tempfile", "toml", ] @@ -541,8 +558,8 @@ dependencies = [ "predicates", "pretty_assertions", "serde", + "serde-saphyr", "serde_json", - "serde_yaml", "tempfile", "tokio", "tracing", @@ -1225,6 +1242,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "encoding_rs_io" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba3fe847045ecff794b9c138293a80db914678c453ad63fbf0c6a9eb6e00b22" +dependencies = [ + "encoding_rs", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -2311,6 +2337,16 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "granit-parser" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da55feca5d1625466bae068f9fd4b87892caeae70c3fe671ddb0f50d41aebf37" +dependencies = [ + "arraydeque", + "smallvec", +] + [[package]] name = "h2" version = "0.4.18" @@ -3017,6 +3053,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e94e1e6445d314f972ff7395df2de295fe51b71821694f0b0e1e79c4f12c8577" +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "nonempty" version = "0.12.0" @@ -3682,9 +3724,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "once_cell", @@ -3829,6 +3871,23 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-saphyr" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a253edfdea34032da21af84e16fcc94f4c7a6ba237f9f3bc2dbf43cea8535c44" +dependencies = [ + "annotate-snippets", + "base64", + "encoding_rs_io", + "granit-parser", + "nohash-hasher", + "num-traits", + "serde_core", + "smallvec", + "zmij", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -3884,19 +3943,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "sha1" version = "0.10.7" @@ -4718,16 +4764,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] -name = "unicode-xid" -version = "0.2.6" +name = "unicode-width" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] -name = "unsafe-libyaml" -version = "0.2.11" +name = "unicode-xid" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "untrusted" diff --git a/Cargo.toml b/Cargo.toml index 7c0266579..aaee69c63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,10 @@ authors = [ # entries via `.workspace = true`, so any grammar bump here MUST be # applied to enums/Cargo.toml in lockstep (see recreate-grammars.sh). [workspace.dependencies] +serde-saphyr = { version = "^2.0", default-features = false, features = [ + "serialize", + "deserialize", +] } tree-sitter = "=0.26.13" tree-sitter-bash = "=0.25.1" tree-sitter-c-sharp = "=0.23.5" @@ -378,7 +382,7 @@ globset = "^0.4" # three) to pin the uniform null/omit policy across all of them. Versions # mirror the `big-code-analysis-cli` pins; these crates already sit in the # workspace graph, so listing them here adds no new resolved dependency. -serde_yaml = "^0.9" +serde-saphyr.workspace = true ciborium = "^0.2" toml = "^1.1" diff --git a/big-code-analysis-ast/src/checker.rs b/big-code-analysis-ast/src/checker.rs index 02e3abe9c..c22c00f5c 100644 --- a/big-code-analysis-ast/src/checker.rs +++ b/big-code-analysis-ast/src/checker.rs @@ -475,7 +475,7 @@ pub trait Checker { /// TypeScript, TSX, PHP, Ruby (`super` / `yield` count as branches) /// and Rust (`?` does). /// - /// The relation is not a plain superset everywhere. Four languages + /// The relation is not a plain superset everywhere. Three languages /// depart from it, each in its own direction: /// /// * Tcl and iRules bill a mutator command (`incr`, `append`, @@ -487,15 +487,15 @@ pub trait Checker { /// * Elixir excludes its definition and directive `Call`s (`def`, /// `defmodule`, `alias`, `import`, …) from `branches` while /// counting each `|>` pipeline step. - /// * Perl counts a bareword call only at the outermost dispatch - /// site: in `print shift;` the wrapper - /// `call_expression_with_spaced_args` is the branch and the - /// `print` bareword it holds is skipped, so that bareword is an - /// `is_call` match with no branch behind it. /// - /// The two sets coincide only in C, Objective-C, Go, Python, Lua - /// and Bash, which have neither a separate construction syntax nor - /// a gated branch arm. + /// The two sets coincide in C, Objective-C, Go, Python, Lua and + /// Bash, which have neither a separate construction syntax nor a + /// gated branch arm, and in Perl, whose one gated arm ABC shares + /// with [`is_call_with_code`](Self::is_call_with_code): a bareword + /// is no call of its own as a wrapper's callee (`print` in + /// `print shift;`) or as an auto-quoted key (`(name => 1)`). That + /// gate needs the ancestor chain, so for Perl this byte-less form is + /// only the kind table the override narrows. // `is_call` and each language's ABC branch arm classify the same // nodes through parallel `matches!()` tables, so they drift // silently; the divergence is pinned by @@ -506,6 +506,23 @@ pub trait Checker { fn is_call(_: &Node) -> bool { false } + /// [`is_call`](Self::is_call), given the source bytes and the + /// chain the caller descended through, which the `call` filter + /// hands it. The default forwards to [`is_call`](Self::is_call). + /// + /// Perl overrides it: the grammar nests a call's name inside the + /// call as a bareword call of its own, and parses an auto-quoted + /// hash key as one, so only the position says which words are + /// calls (#1580). As with [`is_string_with_code`](Self::is_string_with_code), + /// a call site reaching for the byte-less spelling reads as correct + /// against every language without an override; + /// `perl_bareword_is_a_call_only_to_the_byteless_spelling` pins that + /// the two really do disagree. + #[inline] + #[must_use] + fn is_call_with_code<'a>(node: &Node<'a>, _code: &[u8], _ancestors: Ancestors<'a, '_>) -> bool { + Self::is_call(node) + } /// Whether `node`, met among a parameter list's children, is /// punctuation or a keyword rather than a parameter (`nargs` skips it). #[inline] @@ -611,6 +628,14 @@ pub trait Checker { } /// Whether `node` is a primitive type name (`int`, `double`, …), /// which Halstead keys by text so each distinct primitive counts once. + /// + /// Precondition: call this only for a node + /// [`Getter::get_op_type_with_code`] has already classified as + /// [`TokenRole::Operator`]. An impl that cannot read the source may + /// answer by kind alone: Groovy's returns `true` for every + /// `identifier` and `type_identifier`, which is right only because + /// its getter bills those kinds as operators for a primitive + /// spelling and nothing else. #[inline] #[must_use] fn is_primitive(_node: &Node) -> bool { @@ -3521,6 +3546,70 @@ mod tests { ); } + /// A Perl bareword is a call to [`Checker::is_call`] and, as a + /// wrapper's callee or an auto-quoted key, not to + /// [`Checker::is_call_with_code`] (#1580). The end-to-end + /// `find --type call` tests observe the result, which is equally + /// consistent with the override never firing; this asserts the + /// disagreement, so a call site reaching for the byte-less spelling + /// is wrong rather than indistinguishable (grammar-dispatch §7). + /// + /// Three structurally independent routes withdraw a bareword — key + /// position, the spaced wrapper's callee, the bracketed wrapper's + /// callee — and two keep one: a bareword argument and a lone + /// bareword (grammar-dispatch §11). The two wrappers are the control + /// for the kind gate: both spellings must still call them. + #[test] + #[cfg(feature = "perl")] + fn perl_bareword_is_a_call_only_to_the_byteless_spelling() { + let code = b"my %h = (name => 1);\nprint shift;\nfoo(1);\nfoo;\n"; + let (mut kept, mut withdrawn, mut other) = (Vec::new(), Vec::new(), 0); + for_each_node_with_chain::(code, |node, chain| { + let known = PerlCode::is_call_with_code(node, code, Ancestors::known(chain)); + assert_eq!( + known, + PerlCode::is_call_with_code(node, code, Ancestors::unknown()), + "known chain and parent climb disagree on {} at row {}", + node.kind(), + node.start_row() + ); + if node.kind_id() != Perl::CallExpressionWithBareword as u16 { + assert_eq!( + known, + PerlCode::is_call(node), + "the override moved a {} node at row {}", + node.kind(), + node.start_row() + ); + other += usize::from(known); + return; + } + assert!( + PerlCode::is_call(node), + "the kind table must still list the bareword kind, or this test is \ + measuring its absence rather than the override" + ); + let text = + String::from_utf8_lossy(&code[node.start_byte()..node.end_byte()]).into_owned(); + if known { + kept.push(text); + } else { + withdrawn.push(text); + } + }); + assert_eq!( + kept, + ["shift", "foo"], + "a bareword argument and a lone bareword are calls" + ); + assert_eq!( + withdrawn, + ["name", "print", "foo"], + "an auto-quoted key and the callee inside either wrapper are not" + ); + assert_eq!(other, 2, "the two wrappers stay calls to both spellings"); + } + /// The value slots of the three constructs whose argument lists hold /// both roles stay string literals, though `is_value_braced_word` /// classifies each construct whole as script-taking diff --git a/big-code-analysis-ast/src/checker/groovy.rs b/big-code-analysis-ast/src/checker/groovy.rs index 62cc67f15..72e43ed5f 100644 --- a/big-code-analysis-ast/src/checker/groovy.rs +++ b/big-code-analysis-ast/src/checker/groovy.rs @@ -43,6 +43,19 @@ impl Checker for GroovyCode { ) } + // A primitive type is a name kind spelled `int`, `void`, … (#1568), + // which only the bytes can tell, and this predicate gets none. It is + // consulted only for a node `GroovyCode::get_op_type_with_code` + // already billed as an operator, and that bills these two kinds as + // one only for a primitive spelling, so the kind alone suffices to + // key each primitive by its text. + fn is_primitive(node: &Node) -> bool { + matches!( + node.kind_id().into(), + Groovy::Identifier | Groovy::TypeIdentifier + ) + } + fn is_closure<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> bool { matches!(node.kind_id().into(), Groovy::Closure) } diff --git a/big-code-analysis-ast/src/checker/objc.rs b/big-code-analysis-ast/src/checker/objc.rs index 7f28559d7..f26c7eeb0 100644 --- a/big-code-analysis-ast/src/checker/objc.rs +++ b/big-code-analysis-ast/src/checker/objc.rs @@ -113,8 +113,17 @@ impl Checker for ObjcCode { impl_is_else_if_parent_clause!(Objc, IfStatement, ElseClause); + // Besides the primitive types, an `identifier` the getter bills is a + // property attribute word (`nonatomic`, `copy`), whose kind is no + // keyword, so keying it by text lists each word. The `nullable` + // type qualifier is keyed by text too, so it and the `nullable` + // property attribute share one `n1` entry rather than one per map + // (#1579). #[inline] fn is_primitive(node: &Node) -> bool { - node.kind_id() == Objc::PrimitiveType + matches!( + node.kind_id().into(), + Objc::PrimitiveType | Objc::Identifier | Objc::Nullable + ) } } diff --git a/big-code-analysis-ast/src/checker/perl.rs b/big-code-analysis-ast/src/checker/perl.rs index cb2fb89f5..7e15efed7 100644 --- a/big-code-analysis-ast/src/checker/perl.rs +++ b/big-code-analysis-ast/src/checker/perl.rs @@ -2,6 +2,7 @@ #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] use super::*; +use crate::lang_helpers::perl::perl_bareword_is_call; impl Checker for PerlCode { fn is_comment(node: &Node) -> bool { @@ -42,6 +43,12 @@ impl Checker for PerlCode { ) } + fn is_call_with_code<'a>(node: &Node<'a>, _code: &[u8], ancestors: Ancestors<'a, '_>) -> bool { + Self::is_call(node) + && (node.kind_id() != Perl::CallExpressionWithBareword as u16 + || perl_bareword_is_call(node, ancestors)) + } + // `NormalComma` (369) is the separator inside a subroutine signature; // the bare `COMMA` (12) it wraps is that node's *child*, so listing // only `COMMA` let every signature separator count as a parameter diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index 85278aae0..1b2ba231d 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -232,16 +232,16 @@ pub trait Getter { /// counts the alphabet a program is written in and has no notion of /// a decision, so a `<` is the same vocabulary entry whether it /// opens a JSX tag, brackets a Lua `` attribute, names a C# - /// `operator <` overload, disambiguates a Kotlin `super`, or - /// delimits a Perl `` readline. Every impl that classifies - /// punctuation at all bills `(`, `{`, `[`, `,`, `;`, `.` and `:` on - /// exactly that reading, for each glyph its language spells. Python - /// and Rust were the last two exceptions, closed by #1486. + /// `operator <` overload, or delimits a Perl `` readline. Every + /// impl that classifies punctuation at all bills `(`, `{`, `[`, `,`, + /// `;`, `.` and `:` on exactly that reading, for each glyph its + /// language spells. Python and Rust were the last two exceptions, + /// closed by #1486. /// /// A *decision* metric gates the same token by role, and is /// supposed to disagree: ABC's `conditions` counts a `<` only under /// a `binary_expression` (#1274, #1275, #1280, #1297), so each of - /// the five constructs above scores zero there while `bca ops` + /// the four constructs above scores zero there while `bca ops` /// still reports the bracket. #1395 settled that the two answers /// are both correct rather than a drift to be reconciled; /// `tests/parity/abc_halstead_bracket_parity.rs` fails if a later diff --git a/big-code-analysis-ast/src/getter/c.rs b/big-code-analysis-ast/src/getter/c.rs index 8fb6520b7..9131eb3bd 100644 --- a/big-code-analysis-ast/src/getter/c.rs +++ b/big-code-analysis-ast/src/getter/c.rs @@ -86,7 +86,34 @@ impl Getter for CCode { // C++-only forms, so leaving them out here would score a // header differently by the grammar that parsed it. The // derivation is on the Cpp arm. - | Struct | Union | Enum | Typedef => TokenRole::Operator, + | Struct | Union | Enum | Typedef + // Storage classes, function specifiers and type qualifiers, + // billed as every class language bills its modifiers + // (#1578). Each is the keyword leaf of an unclassified + // `storage_class_specifier` / `type_qualifier` / + // `alignas_qualifier`, so it bills once, as `const` does in + // a pointer declarator (`int * const p`). The vendor + // spellings (`__inline`, `__restrict__`, `__thread`, + // `__volatile__`, …) are kinds of their own and bill under + // their own name; `__volatile__` the grammar emits only as a + // GNU `asm` qualifier, where `volatile` bills too. `auto` + // is a storage class in C, the only place this grammar + // emits it. `__extension__` bills in an expression too, + // where it is the prefix of an `extension_expression`. + // Attributes (`__attribute__`, `__declspec`, `[[…]]`) and + // the MS calling conventions stay unbilled: they annotate a + // declaration rather than spell part of it. `cpp.rs` copies + // this list, though `restrict` and `noreturn` are no C++ + // keywords: `int restrict = 0;` is a valid C++ variable, as + // `int noreturn = 1;` is in C without ``. Both + // grammars still parse the name as the qualifier before a + // zero-width `identifier`, so the keyword bills and no + // operand does — the grammar decides that, not this arm, as + // `objc.rs` notes for `nullable`. + | Static | Extern | Register | Auto | Inline | Inline2 | Inline3 | Forceinline + | ThreadLocal | Thread | Const | Constexpr | Volatile | Volatile2 | Restrict | Restrict2 + | Atomic | Noreturn | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension + => TokenRole::Operator, // `CharLiteral` joins the operand list here for the whole // C family — `cpp.rs`, `mozcpp.rs` and `objc.rs` carry the // same arm and point back at this note (#1316). Before it a diff --git a/big-code-analysis-ast/src/getter/cpp.rs b/big-code-analysis-ast/src/getter/cpp.rs index 89ac98ee3..60821e3c2 100644 --- a/big-code-analysis-ast/src/getter/cpp.rs +++ b/big-code-analysis-ast/src/getter/cpp.rs @@ -203,6 +203,34 @@ impl Getter for CppCode { // spellings of `template ` still score alike, now // both billed. | Class | Struct | Union | Enum | Namespace | Template | Typename | Typedef | Using + // C's storage classes, function specifiers and type + // qualifiers (#1578; the derivation is on the C arm), plus + // C++'s own modifiers: `virtual`, `mutable`, `constinit`, + // `consteval`, `explicit`, `friend`, the access specifiers + // and the `override` / `final` virt-specifiers, and the + // `noexcept` of an exception specification. Each is the + // keyword leaf of an unclassified wrapper + // (`storage_class_specifier`, `type_qualifier`, + // `virtual_specifier`, `access_specifier`, + // `explicit_function_specifier`, the `noexcept` node that + // `Noexcept` names and `Noexcept2` is the leaf of) or sits + // bare in its declaration (`virtual`, `friend`), so each + // bills once. An access label bills as Java's `public` does, + // beside the `:` that already billed. `const` bills in a + // member-function trailer and a pointer declarator alike, + // and `constexpr` in `if constexpr`, which it qualifies. + // + // `override` and `final` are contextual: used as a name + // (`int override = 1;`) they parse as an `identifier` and + // stay operands, so they need no guard. `auto` is not here: + // in C++ it is a placeholder type, not a storage class, and + // the grammar emits it only under + // `placeholder_type_specifier`. + | Static | Extern | Register | Inline | Inline2 | Inline3 | Forceinline + | ThreadLocal | Thread | Const | Constexpr | Volatile | Volatile2 | Restrict | Restrict2 + | Atomic | Noreturn | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension + | Virtual | Mutable | Constinit | Consteval | Explicit | Friend | Public | Private + | Protected | Override | Final | Noexcept2 => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only diff --git a/big-code-analysis-ast/src/getter/csharp.rs b/big-code-analysis-ast/src/getter/csharp.rs index 000249167..cfb5aa0d4 100644 --- a/big-code-analysis-ast/src/getter/csharp.rs +++ b/big-code-analysis-ast/src/getter/csharp.rs @@ -3,10 +3,13 @@ use super::*; +// A function pointer type's `<>` (`delegate*`) encloses +// its parameter and return types (#1567). const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { lists: &[ Csharp::TypeArgumentList as u16, Csharp::TypeParameterList as u16, + Csharp::FunctionPointerType as u16, ], openers: &[Csharp::LT as u16], closers: &[Csharp::GT as u16], @@ -22,18 +25,30 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { // a call or indexer argument without `out`. `F(out List x)`, // `(List p, int q) = …` and `o is List x` keep their pair. A // member access before the `<` (`F(x.a < b, c > d)`) wraps the generic -// name in `qualified_name`s, which the climb passes. The `>=` spelling +// name in a `qualified_name`, whose final name alone can be the misparse. +// Generic qualifiers keep their type brackets. The `>=` spelling // (`F(a < b, c >= d)`) parses as an assignment instead and is not undone. fn csharp_comparison_misparse<'a>(list: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { let mut up = ancestors.iter(list).map(|(node, _)| node); - if up + let Some(name) = up .next() - .is_none_or(|name| name.kind_id() != Csharp::GenericName as u16) - { + .filter(|name| name.kind_id() == Csharp::GenericName as u16) + else { return false; - } - let mut up = up.skip_while(|node| node.kind_id() == Csharp::QualifiedName as u16); - let (Some(declaration), Some(argument), Some(arguments)) = (up.next(), up.next(), up.next()) + }; + let declaration = match up.next() { + Some(qualified) if qualified.kind_id() == Csharp::QualifiedName as u16 => { + if qualified + .child_by_field_name("name") + .is_none_or(|last| last.id() != name.id()) + { + return false; + } + up.next() + } + parent => parent, + }; + let (Some(declaration), Some(argument), Some(arguments)) = (declaration, up.next(), up.next()) else { return false; }; @@ -46,6 +61,35 @@ fn csharp_comparison_misparse<'a>(list: &Node<'a>, ancestors: Ancestors<'a, '_>) ) } +// Whether `node` is the `,` opening an interpolation's alignment +// (`$"{x,8}"`) or the `:` opening its format string (`$"{x:F2}"`). +// Both delimit the field, as the `{` / `}` braces do, which #1314 left +// unbilled in every interpolating language; Python's format-spec `:` +// went the same way in #1486 (#1573). The guard reads the parent, so a +// ternary `:`, a named-argument `:` or an argument-list `,` written +// inside the hole (`$"{f(a, b)}"`) keeps billing. +// +// The alignment *value* is a `constant_expression` and walks as one: +// `{x,8}` bills `8` and `{x,-W}` bills `-` and `W`. That is where C# +// and Python differ: a Python literal width (`f"{x:>10}"`) is +// format-spec text and bills nothing, while C#'s `8` is an +// `integer_literal` token. An expression width bills in both +// (`f"{x:>{w}}"` bills `w`). The format string after `:` (`F2`) is +// string content and has no node to bill. +// +// On invalid input with no expression before the delimiter +// (`$"{x,}"`, `$"{:F2}"`) error recovery leaves the `,` / `:` outside +// any clause node, so it bills. Valid C# always has both, so this is +// documented rather than tested (grammar-dispatch §6). +fn is_interpolation_clause_delimiter<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let clause = match node.kind_id().into() { + Csharp::COLON => Csharp::InterpolationFormatClause, + Csharp::COMMA => Csharp::InterpolationAlignmentClause, + _ => return false, + }; + ancestors.parent_has_kind(node, clause as u16) +} + impl Getter for CsharpCode { fn get_space_kind(node: &Node) -> SpaceKind { use Csharp::*; @@ -91,7 +135,9 @@ impl Getter for CsharpCode { fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Csharp::*; - if GENERIC_ANGLES.is_closer(node, ancestors) { + if GENERIC_ANGLES.is_closer(node, ancestors) + || is_interpolation_clause_delimiter(node, ancestors) + { return TokenRole::Unknown; } diff --git a/big-code-analysis-ast/src/getter/generic_angles.rs b/big-code-analysis-ast/src/getter/generic_angles.rs index ff67d09ad..97eba85aa 100644 --- a/big-code-analysis-ast/src/getter/generic_angles.rs +++ b/big-code-analysis-ast/src/getter/generic_angles.rs @@ -10,18 +10,23 @@ use crate::{Ancestors, Node}; pub(crate) const ANGLE_PAIR: &str = "<>"; /// The kinds one grammar spells its generic angle brackets with: the -/// list nodes whose `<` / `>` delimit type arguments or parameters -/// rather than compare, and the bracket tokens themselves. +/// nodes whose `<` / `>` enclose types rather than compare, and the +/// bracket tokens themselves. /// -/// A list's brackets are one bracket pair, billed like `()`, `[]` and -/// `{}` (#1395): the opener carries the pair as [`ANGLE_PAIR`] and the -/// closer bills nothing (#1559). The grammars give the brackets the -/// same kind ids as the comparison operators, so only the parent tells -/// the two apart, and the opener is keyed by text rather than by kind -/// to keep it out of the comparison `<`'s vocabulary entry. +/// Angle brackets that enclose types are one bracket pair, billed like +/// `()`, `[]` and `{}` (#1395): the opener carries the pair as +/// [`ANGLE_PAIR`] and the closer bills nothing (#1559). The grammars +/// give the brackets the same kind ids as the comparison operators, so +/// only the parent tells the two apart, and the opener is keyed by text +/// rather than by kind to keep it out of the comparison `<`'s +/// vocabulary entry. pub(crate) struct GenericAngleKinds { - /// The list nodes: type-argument, type-parameter and template - /// lists, plus each grammar's other spelling of a generic list. + /// The nodes whose brackets enclose types: type-argument, + /// type-parameter and template lists, each grammar's other spelling + /// of a generic list, and the single-type brackets of a Rust + /// qualified path (`::f`), a C# function pointer + /// (`delegate*`) and a Kotlin `super` (#1567). Each + /// has `<` / `>` only as its own brackets. pub(crate) lists: &'static [u16], /// The `<` kinds, with any alias the grammar declares for it. pub(crate) openers: &'static [u16], diff --git a/big-code-analysis-ast/src/getter/groovy.rs b/big-code-analysis-ast/src/getter/groovy.rs index a93d2a35d..cb289099e 100644 --- a/big-code-analysis-ast/src/getter/groovy.rs +++ b/big-code-analysis-ast/src/getter/groovy.rs @@ -20,6 +20,75 @@ const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { is_misparse: never_misparsed, }; +/// Whether a contextual keyword leaf (`trait`, `record`, `sealed`, +/// `permits`) sits directly under the construct it introduces. +/// +/// Valid Groovy may use any of the four as a name. Inside a class the +/// grammar parses that name as an `identifier`, but a script-level +/// statement that opens with one (`record = 3`, `String record`, +/// `void record() {}`, `sealed.call()`) fails into an `ERROR` holding +/// the keyword leaf. Billing the leaf there would make a variable an +/// operator (#1552, #1565). The pinned grammar also misreads a *nested* +/// `sealed class` as a field typed `sealed`, so that `sealed` bills as +/// a type-name operand. +fn contextual_keyword_owns<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + use Groovy::*; + + ancestors.parent(node).is_some_and(|parent| { + matches!( + (node.kind_id().into(), parent.kind_id().into()), + (Trait, TraitDeclaration) + | (Record, RecordDeclaration) + | (Permits, PermitsClause) + | ( + Sealed, + ClassDeclaration | InterfaceDeclaration | TraitDeclaration + ) + ) + }) +} + +/// Java's eight primitive type keywords and `void`, which Groovy +/// reserves as Java does. +const PRIMITIVE_TYPES: &[&str] = &[ + "boolean", "byte", "char", "double", "float", "int", "long", "short", "void", +]; + +/// Whether `node` spells a primitive type, which bills as a text-keyed +/// operator as Java's primitive keywords do (#1568). +/// +/// The pinned grammar has no primitive node: a declared type is a +/// `type_identifier` (`int x = 1`, `void m()`), and a class literal's +/// receiver (`int.class`) or an uninitialised local (`int t`, which +/// parses as two expression statements) is a plain `identifier`. Groovy +/// reserves these keywords, so a name can spell one only as a member +/// selector after a `.`-family operator (`x.int`, `x?.int`, `x::int`) or +/// as a map or named-argument label (`[int: 1]`, `f(int: 1)`); those +/// stay operands. +fn spells_primitive_type<'a>(node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>) -> bool { + use Groovy::*; + + if !matches!(node.kind_id().into(), Identifier | TypeIdentifier) + || !node_text(code, node).is_some_and(|text| PRIMITIVE_TYPES.contains(&text)) + { + return false; + } + ancestors.parent(node).is_none_or(|parent| { + let is_first_child = parent.child(0).is_some_and(|first| first.id() == node.id()); + match parent.kind_id().into() { + FieldAccess + | SafeNavigationExpression + | SafeChainDotExpression + | SpreadDotExpression + | DirectFieldAccessExpression + | MethodPointerExpression + | MethodReferenceExpression => is_first_child, + MapEntry | NamedArgument => !is_first_child, + _ => true, + } + }) +} + impl Getter for GroovyCode { /// Names the space, synthesising one for constructs that carry no /// name token (#1184). @@ -103,9 +172,11 @@ impl Getter for GroovyCode { if GENERIC_ANGLES.is_closer(node, ancestors) { return TokenRole::Unknown; } - // Mirrors `JavaCode`'s minimal classification — modifiers - // (`Public`, `Static`, …) and module keywords (`Package`, - // `Import`, …) are excluded. The type-declaration keywords + // Mirrors `JavaCode`'s classification: modifiers are billed + // (#1565) and module keywords (`Package`, `Import`, …) are + // excluded. The grammar has no `modifiers` wrapper — each + // modifier leaf sits directly under its declaration — so each + // bills once. The type-declaration keywords // (`class`, `interface`, `trait`, `enum`, `record`) are billed, // as Java's are since #1552: no `*Declaration` wrapper is // classified, so the keyword leaf is the only node that can @@ -178,18 +249,17 @@ impl Getter for GroovyCode { Super if ancestors.parent_has_kind(node, Wildcard as u16) => TokenRole::Operator, Super => TokenRole::Operand, - // `trait` and `record` are contextual keywords, so valid - // Groovy may use either as a name. Inside a class the grammar - // parses that name as an `identifier`, but at script level it - // fails `record = 3`, `String record` and `void record() {}` - // into an `ERROR` holding the keyword leaf. Billing the leaf - // there would make a variable an operator, so it bills only - // under the declaration it introduces (#1552). - Trait if ancestors.parent_has_kind(node, TraitDeclaration as u16) => { + // `trait`, `record`, `sealed` and `permits` are contextual + // keywords, so each bills only under the construct it + // introduces (`contextual_keyword_owns`); elsewhere it falls + // through to `Unknown`. + Trait | Record | Sealed | Permits if contextual_keyword_owns(node, ancestors) => { TokenRole::Operator } - Record if ancestors.parent_has_kind(node, RecordDeclaration as u16) => { - TokenRole::Operator + // `import static`'s `static` bills nothing, as `import` does + // not; Java makes the same call. + Static if ancestors.parent_has_kind(node, ImportDeclaration as u16) => { + TokenRole::Unknown } // Control-flow + keyword operators (mirrors Java's set, @@ -198,6 +268,11 @@ impl Getter for GroovyCode { If | Else | Switch | Case | Try | Catch | Throw | Throws | For | While | Continue | Break | Do | Finally | New | Return | Default | Abstract | Assert | Instanceof | Extends | Final | Implements | Transient | Synchronized | Def | In | As + // The rest of the modifiers (#1565). `non-sealed` cannot be + // spelled as a name; `non-sealed` as a subtraction parses as + // `non`, `-` and an identifier `sealed`. + | Public | Private | Protected | Static | Native | Strictfp | Volatile + | NonDASHsealed // Type-declaration keywords (#1552). Unlike Java's, `class` // needs no class-literal guard: `String.class` parses as a // `field_access` whose `class` is a plain `identifier`. @@ -269,6 +344,20 @@ impl Getter for GroovyCode { } } + // Only the bytes tell `int` from `Integer`, so the byte-less + // `get_op_type` above still answers `Operand` for a primitive; the + // walk calls this spelling (grammar-dispatch section 7). + fn get_op_type_with_code<'a>( + node: &Node<'a>, + code: &[u8], + ancestors: Ancestors<'a, '_>, + ) -> TokenRole { + if spells_primitive_type(node, code, ancestors) { + return TokenRole::Operator; + } + Self::get_op_type(node, ancestors) + } + fn get_operator_spelling<'a>( node: &Node<'a>, ancestors: Ancestors<'a, '_>, diff --git a/big-code-analysis-ast/src/getter/java.rs b/big-code-analysis-ast/src/getter/java.rs index b54391626..e2b93b2dc 100644 --- a/big-code-analysis-ast/src/getter/java.rs +++ b/big-code-analysis-ast/src/getter/java.rs @@ -96,23 +96,22 @@ impl Getter for JavaCode { // https://www.geeksforgeeks.org/software-engineering-halsteads-software-metrics/?msclkid=5e181114abef11ecbb03527e95a34828 match node.kind_id().into() { // `class` declares a class (billed below, #1552) except as the - // tail of a class literal (`C.class`): that `class_literal` is - // already one operand keyed by its whole text, so billing its - // leaf too would pay the same bytes twice (grammar-dispatch - // section 5, the compound-leaf guard). - Class if ancestors.parent_has_kind(node, ClassLiteral as u16) => TokenRole::Unknown, - // A type name is an operand (#1560, the operand arm below) - // except inside a class literal, whose whole text is already - // one operand — the `class` guard above, for the same - // reason. The type sits there directly (`Foo.class`) or - // under an array or scoped type (`Foo[].class`, - // `java.util.List.class`), so the climb skips those two. - TypeIdentifier - if ancestors - .iter(node) - .map(|(ancestor, _)| ancestor.kind_id()) - .find(|&id| id != ArrayType as u16 && id != ScopedTypeIdentifier as u16) - == Some(ClassLiteral as u16) => + // tail of a class literal (`C.class`), where it is an operand: + // JLS §15.8.2 makes the literal a primary expression, not a + // literal (§3.10), so its parts bill as they would anywhere + // else — the type name an operand, the primitive or `void` + // and the `.` / `[]` operators — and the `class` tail is the + // operand they act on, as Kotlin's and PHP's `Foo::class` and + // Groovy's `Foo.class` bill theirs (#1566). + Class if ancestors.parent_has_kind(node, ClassLiteral as u16) => TokenRole::Operand, + // `static` is a modifier (below, #1565) except in an import + // or module directive (`import static`, `requires static`), + // whose own keyword — `import`, `requires` — bills nothing + // here, so neither does the word it qualifies. + Static + if ancestors.parent(node).is_some_and(|p| { + matches!(p.kind_id().into(), ImportDeclaration | RequiresModifier) + }) => { TokenRole::Unknown } @@ -122,6 +121,18 @@ impl Getter for JavaCode { // Operator: keywords | New | Return | Default | Abstract | Assert | Instanceof | Extends | Final | Implements | Transient | Synchronized | VoidType + // Operator: the rest of the modifiers, billed as C# and Kotlin + // bill theirs (#1565). Each is its keyword leaf, one kind + // apiece; the `modifiers` wrapper stays unlisted, so a + // modifier bills once and an annotation inside it adds only + // its own tokens. `default` above is one kind for a `switch` + // label and an interface method's modifier. `sealed` and + // `permits` (`Permits2`; `Permits` is the clause) are + // contextual, and the grammar parses each used as a name + // (`var sealed = 1`, `int permits`) as an `identifier`, so + // they need no guard; `non-sealed` cannot be a name. + | Public | Private | Protected | Static | Native | Strictfp | Volatile | Sealed + | NonDASHsealed | Permits2 // Operator: type-declaration keywords, billed as C# bills // its own (#1552). Only the leaves: no `*_declaration` // wrapper is listed, so a declaration bills its keyword once. @@ -166,10 +177,11 @@ impl Getter for JavaCode { // Go, Groovy and TypeScript bill type names: a field or // parameter type, an `implements` / `extends` entry, a type // parameter, a generic argument, each segment of - // `java.util.List` (the `.` bills as itself). It is the one + // `java.util.List` (the `.` bills as itself), and since #1566 + // the type of a class literal (`Foo.class`). It is the one // type-name kind and no listed node wraps it, so each name // bills once. - Identifier | TypeIdentifier | NullLiteral | ClassLiteral | True | False + Identifier | TypeIdentifier | NullLiteral | True | False | StringLiteral | CharacterLiteral | HexIntegerLiteral | OctalIntegerLiteral | BinaryIntegerLiteral | DecimalIntegerLiteral | HexFloatingPointLiteral | DecimalFloatingPointLiteral | This => { @@ -214,6 +226,9 @@ impl Getter for JavaCode { Java::LBRACK => "[]", Java::LBRACE => "{}", Java::VoidType => "void", + // The kind is `boolean_type`; spelled as written, as Groovy's + // text-keyed primitives are (#1568). + Java::BooleanType => "boolean", _ => typ.into(), } } diff --git a/big-code-analysis-ast/src/getter/javascript.rs b/big-code-analysis-ast/src/getter/javascript.rs index c6c1c787f..a148fff99 100644 --- a/big-code-analysis-ast/src/getter/javascript.rs +++ b/big-code-analysis-ast/src/getter/javascript.rs @@ -56,9 +56,14 @@ impl Getter for JavascriptCode { // TypeScript, which shares this macro, has no JSX (#1395). // `jsx:` bills element text and its entities as operands, for the // same reason (#1483). + // + // `Staticget` is one token spelling `static get` followed by a + // newline, the static getter whose name starts the next line. It is + // billed beside the shared `static` (#1563) so that form bills a + // modifier rather than nothing; being one node it is one operator. impl_js_family_get_op_type!( Javascript, - op_extras: [OptionalChain, LTSLASH, SLASHGT], + op_extras: [OptionalChain, LTSLASH, SLASHGT, Staticget], operand_extras: [Identifier2, String2], jsx: [JsxText, HtmlCharacterReference], ); diff --git a/big-code-analysis-ast/src/getter/js_family.rs b/big-code-analysis-ast/src/getter/js_family.rs index dc9aea021..4c963d90d 100644 --- a/big-code-analysis-ast/src/getter/js_family.rs +++ b/big-code-analysis-ast/src/getter/js_family.rs @@ -24,6 +24,10 @@ // are extras rather than members of the shared arm because // TypeScript is the one grammar of the four with no JSX and // therefore no such variants to name. +// * JavaScript / MozJS: `Staticget`, the one-token `static get` +// these grammars alone have (#1563). +// * TypeScript / TSX: the declaration keywords, type-level operator +// keywords and member modifiers listed at each invocation. // // `$operand_extras` per language: // * JavaScript / MozJS / TSX: `Identifier2`, `String2` — anonymous @@ -237,6 +241,14 @@ macro_rules! impl_js_family_get_op_type { // groups across languages, skewing n1/n2 for accessor-heavy // code (#695). | Set | Get + // `static` is a class-member modifier, billed as C# and + // Kotlin bill theirs (#1563; Java's and Groovy's are + // #1565). It is the bare leaf in a field, a method and + // a `static { … }` block alike, with no modifier + // wrapper, and one kind in all four grammars. Used as a + // name (`obj.static`, `static()`, a field named + // `static`) the grammar emits an identifier instead. + | Static $(| $op_extra)* => TokenRole::Operator, // `Regex` is the literal's own node and contributes one // operand, the way Ruby's `Regex` and Elixir's `Sigil` diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index 1255c0a10..1b879404f 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -157,15 +157,12 @@ fn kotlin_modifier_was_written_as_one<'a>(node: &Node<'a>, ancestors: Ancestors< // `function_value_parameters`, so the sibling is found without a scan // from the list's start: that made a long parameter list quadratic. fn kotlin_next_token_is_inserted<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - let mut current = *node; - ancestors - .iter(node) - .find_map(|(parent, _)| { - let next = parent - .children_after(¤t) - .find(|sibling| !sibling.as_tree_sitter().is_extra()); - current = parent; - next + std::iter::once((*node, ancestors)) + .chain(ancestors.iter(node)) + .find_map(|(level, level_ancestors)| { + level_ancestors + .siblings_after(&level) + .find(|sibling| !sibling.as_tree_sitter().is_extra()) }) .is_some_and(|mut next| { while let Some(first) = next.child(0) { @@ -175,10 +172,14 @@ fn kotlin_next_token_is_inserted<'a>(node: &Node<'a>, ancestors: Ancestors<'a, ' }) } -// `super` (a `super_expression`) names a supertype rather than -// listing type arguments, so its brackets stay `<` and `>`. +// A qualified `super` (a `super_expression`) encloses the supertype +// it names (#1567). const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { - lists: &[Kotlin::TypeArguments as u16, Kotlin::TypeParameters as u16], + lists: &[ + Kotlin::TypeArguments as u16, + Kotlin::TypeParameters as u16, + Kotlin::SuperExpression as u16, + ], openers: &[Kotlin::LT as u16], closers: &[Kotlin::GT as u16], is_misparse: never_misparsed, @@ -340,6 +341,31 @@ impl Getter for KotlinCode { }; } + // The accessor keywords, billed as C# bills its `get` / `set` + // (#1564), but only directly under their `getter` / `setter`, + // which stay unlisted. The same leaves are the `@get:` / `@set:` + // use-site targets, which bill nothing, like `@field:` and + // `@property:`; and where kotlin-ng misreads a call `get(1)` after + // a local `val f = 6`, an `ERROR` holds the leaf. Used as a name + // otherwise (`val get = 1`, `x.set(v)`, `fun get()`) the word + // parses as an identifier. The grammar also reads a local + // `val f = 6` followed by `set = 2` as a setter, which bills + // `set`. `field` in an accessor body is an identifier, so it is an + // operand like any other variable. An early return for the same + // reason as the modifiers'. + let accessor = match kind { + Get => Some(Kotlin::Getter), + Set => Some(Setter), + _ => None, + }; + if let Some(accessor) = accessor { + return if ancestors.parent_has_kind(node, accessor as u16) { + TokenRole::Operator + } else { + TokenRole::Unknown + }; + } + match kind { // Operator: control flow keywords If | Else | When | For | While | Do | Try | Catch | Finally | Throw | Return @@ -348,8 +374,15 @@ impl Getter for KotlinCode { // keyword leaf beside `class` and `object` (#1556); its // `class_declaration` wrapper stays unlisted. `companion` is // the one modifier billed here; see the early return above. + // `typealias`, `init`, `where` and `by` bill as C# bills + // `where` and TypeScript `type` (#1564), each the leaf of its + // `type_alias` / `anonymous_initializer` / `type_constraints` + // / `property_delegate` or `explicit_delegation` wrapper. + // `typealias` is a hard keyword; the other three, used as + // names (`val by = 1`, `where(x)`, `init.f()`), parse as + // identifiers, so the leaf exists only in its construct. | Class | Interface | Fun | Object | Val | Var | In | Is | As | AsQMARK | BANGis - | BANGin | Constructor | Companion + | BANGin | Constructor | Companion | Typealias | Init | Where | By // Operator: brackets, separators, terminators | SEMI | COMMA | COLONCOLON | DOT | LBRACE | LBRACK | LPAREN // Operator: assignment and arithmetic diff --git a/big-code-analysis-ast/src/getter/mozcpp.rs b/big-code-analysis-ast/src/getter/mozcpp.rs index c822ec32b..e701395a4 100644 --- a/big-code-analysis-ast/src/getter/mozcpp.rs +++ b/big-code-analysis-ast/src/getter/mozcpp.rs @@ -136,6 +136,14 @@ impl Getter for MozcppCode { // The type-declaration keywords — the derivation is on the // Cpp twin (#1552, #1557). | Class | Struct | Union | Enum | Namespace | Template | Typename | Typedef | Using + // The declaration modifiers — the derivation is on the Cpp + // twin (#1578). The Gecko `MOZ_*` annotation macros stay + // unbilled, as attributes do. + | Static | Extern | Register | Inline | Inline2 | Inline3 | Forceinline + | ThreadLocal | Thread | Const | Constexpr | Volatile | Volatile2 | Restrict | Restrict2 + | Atomic | Noreturn | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension + | Virtual | Mutable | Constinit | Consteval | Explicit | Friend | Public | Private + | Protected | Override | Final | Noexcept2 => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only diff --git a/big-code-analysis-ast/src/getter/mozjs.rs b/big-code-analysis-ast/src/getter/mozjs.rs index 9486d4483..b21b6d3a8 100644 --- a/big-code-analysis-ast/src/getter/mozjs.rs +++ b/big-code-analysis-ast/src/getter/mozjs.rs @@ -55,10 +55,11 @@ impl Getter for MozjsCode { // `$op_extras` note: they are per-language extras because // TypeScript, which shares this macro, has no JSX (#1395). // `jsx:` bills element text and its entities as operands, for the - // same reason (#1483). + // same reason (#1483). `Staticget`: see the JavaScript invocation + // (#1563). impl_js_family_get_op_type!( Mozjs, - op_extras: [OptionalChain, LTSLASH, SLASHGT], + op_extras: [OptionalChain, LTSLASH, SLASHGT, Staticget], operand_extras: [Identifier2, String2], jsx: [JsxText, HtmlCharacterReference], ); diff --git a/big-code-analysis-ast/src/getter/objc.rs b/big-code-analysis-ast/src/getter/objc.rs index c94315d4a..248aa4113 100644 --- a/big-code-analysis-ast/src/getter/objc.rs +++ b/big-code-analysis-ast/src/getter/objc.rs @@ -40,6 +40,19 @@ fn objc_argument_list_misparse(list: &Node, _ancestors: Ancestors) -> bool { list.kind_id() == Objc::ArgumentList2 as u16 && list.has_error() } +// Whether `node` is the attribute word of `@property (nonatomic, copy, +// getter=isOn)`. The grammar has no keyword for any attribute: each is +// an `identifier`, the first child of its `property_attribute`, so the +// word is told from a name by position alone. The `isOn` of +// `getter=isOn` is a later child and stays an operand, the selector it +// names. `ObjcCode::is_primitive` keys the word by its text (#1579). +fn is_property_attribute_name(node: &Node, ancestors: Ancestors) -> bool { + ancestors.parent(node).is_some_and(|attribute| { + attribute.kind_id() == Objc::PropertyAttribute as u16 + && attribute.start_byte() == node.start_byte() + }) +} + impl Getter for ObjcCode { fn get_func_space_name<'a, 'tree>( node: &Node<'tree>, @@ -144,6 +157,40 @@ impl Getter for ObjcCode { // arm: it parses as an `@` token, already billed here, and a // separate `class` leaf, so the directive already bills one // operator. + // + // From `Static` onwards come the declaration modifiers + // (#1579), billed as every class language bills its + // modifiers: first C's storage classes, function specifiers + // and type qualifiers, the same list as `c.rs` (#1578), so a + // `.m` scores a C declaration as a `.c` does; then ObjC's + // own. Those are the ARC ownership qualifiers (`__weak`, + // `__strong`, `__block`, …) and `__bridge` casts, the + // nullability qualifiers in all three spellings + // (`nullable`, `_Nullable`, `__nullable`), `__kindof`, the + // `__covariant` / `__contravariant` variance of a generic + // parameter, the `_Complex` type keyword, which this grammar + // files as a qualifier, the `in` / `out` / `inout` / + // `bycopy` / `byref` / `oneway` method-type qualifiers (`in` + // already billed above), the `@public` / `@private` / + // `@protected` / `@package` ivar visibility labels and the + // `@optional` / `@required` protocol section labels. Each is + // the keyword leaf of an unclassified `storage_class_specifier` + // / `type_qualifier` / `protocol_qualifier` / + // `visibility_specification` or sits bare in its parent, so + // it bills once. None has a numeric-suffix alias. + // + // Left unbilled: the Apple SDK macros this grammar also lists + // as storage classes and qualifiers (`NS_INLINE`, + // `FOUNDATION_EXPORT`, `IBOutlet`, `__unused`, + // `__ptrauth_objc_*`, …), as Mozcpp's `MOZ_*` macros are, + // since each annotates a declaration and a `.c` file parses + // the same spelling as a name; and GNU's `__real` / + // `__imag`, complex-part operators the grammar misfiles as + // qualifiers. A variable named `nullable` parses as the + // qualifier before a MISSING name and bills it, as C's + // `int noreturn = 1;` bills `noreturn`: the grammar decides + // that, not this arm. A property attribute has no keyword + // kind at all; see `is_property_attribute_name`. DOT | LPAREN | LPAREN2 | COMMA | STAR | GTGT | COLON | SEMI | Return | Break | Continue | If | Else | Switch | Case | Default | For | While | Goto | Do | EQ | AMPAMP | PIPEPIPE | DASH | DASHDASH | DASHGT | PLUS | PLUSPLUS | SLASH | PERCENT @@ -152,9 +199,16 @@ impl Getter for ObjcCode { | CARETEQ | PIPEEQ | LBRACK | LBRACE | QMARK | PrimitiveType | TypeSpecifier | Sizeof | Signed | Unsigned | Long | Short | In | AT | ATtry | ATcatch | ATfinally | ATthrow | ATsynchronized | ATautoreleasepool | ATselector | ATencode | Struct - | Union | Enum | Typedef | ATinterface | ATimplementation | ATprotocol => { - TokenRole::Operator - } + | Union | Enum | Typedef | ATinterface | ATimplementation | ATprotocol | Static + | Extern | Register | Auto | Inline | Inline2 | Inline3 | Forceinline | ThreadLocal + | Thread | Const | Constexpr | Volatile | Volatile2 | Restrict | Restrict2 | Atomic + | Noreturn | Noreturn2 | Nonnull | Alignas | Alignas2 | Extension | Const2 + | Complex | Complex2 | Nullable | Nullable2 | Nullable3 | NullableResult + | NullUnspecified | Nonnull2 | Autoreleasing | Block | Strong | UnsafeUnretained + | Weak | Bridge | BridgeRetained | BridgeTransfer | Kindof | Covariant + | Contravariant | Out | Inout | Bycopy | Byref | Oneway | ATpublic | ATprivate + | ATprotected | ATpackage | AToptional | ATrequired => TokenRole::Operator, + Identifier if is_property_attribute_name(node, ancestors) => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only // classified node in a character literal, so it bills one diff --git a/big-code-analysis-ast/src/getter/perl.rs b/big-code-analysis-ast/src/getter/perl.rs index 12228f275..d4f0155e9 100644 --- a/big-code-analysis-ast/src/getter/perl.rs +++ b/big-code-analysis-ast/src/getter/perl.rs @@ -3,7 +3,7 @@ use super::*; use crate::lang_helpers::perl::{ - perl_and_is_operator, perl_dash_signs_keyword_key, perl_file_test_is_dash_key, + perl_and_is_operator, perl_bareword_key_sign, perl_dash_signs_key, perl_file_test_is_dash_key, perl_is_dash_key_word, perl_key_is_signed, perl_not_is_key, }; @@ -258,6 +258,13 @@ impl Getter for PerlCode { /// keys `(- not => 1)` as `"-not"` too. The sign is read off the tree /// (`perl_key_is_signed`), not the byte before the key, which can /// equally be an infix `-` or the end of a comment. + /// + /// Any other `-bareword` key (`(-name => 1)`, `$h{-name}`) bills its + /// word spelled from the sign (#1572). That is a source slice, so a + /// sign spaced off its word (`(- name => 1)`, rare) keeps the space, + /// and a comment between the two (`(- # c⏎ name => 1)`) is carried + /// along in it: one operand, distinct from `name`, but not merged + /// with `-name`, and listed by `bca ops` as written. fn get_operand_id<'a>( node: &Node<'a>, code: &'a [u8], @@ -274,6 +281,8 @@ impl Getter for PerlCode { } Perl::And if perl_key_is_signed(node, ancestors) => b"-and", Perl::Not if perl_key_is_signed(node, ancestors) => b"-not", + Perl::Identifier => perl_bareword_key_sign(node, ancestors) + .map_or(&code[start..end], |dash| &code[dash.start_byte()..end]), _ => &code[start..end], } } @@ -342,8 +351,11 @@ fn auto_quoted_key_role<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> Op // one operand spelling the whole key, and the word inside // bills nothing. A `-and` / `-not` key splits instead into // a `-` and the keyword, which the arms above bill as the - // key; the `-` is part of it and bills nothing. See - // `perl_file_test_is_dash_key`. + // key, and any other `-bareword` key (`(-name => 1)`) into a + // `-` and a bareword call, whose `identifier` bills the key + // (#1572, the local workaround for upstream #1571); in both + // the `-` is part of the key and bills nothing. See + // `perl_file_test_is_dash_key` and `perl_bareword_is_key`. P::FileHandleOperator if perl_file_test_is_dash_key(node, ancestors) => TokenRole::Operand, P::Identifier if ancestors @@ -353,13 +365,7 @@ fn auto_quoted_key_role<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> Op { TokenRole::Unknown } - P::DASH - if ancestors - .parent(node) - .is_some_and(|parent| perl_dash_signs_keyword_key(node, &parent)) => - { - TokenRole::Unknown - } + P::DASH if perl_dash_signs_key(node, ancestors) => TokenRole::Unknown, _ => return None, }; Some(role) diff --git a/big-code-analysis-ast/src/getter/rust.rs b/big-code-analysis-ast/src/getter/rust.rs index be49e11fc..df7d21b47 100644 --- a/big-code-analysis-ast/src/getter/rust.rs +++ b/big-code-analysis-ast/src/getter/rust.rs @@ -5,16 +5,19 @@ use super::*; // `for_lifetimes` (`for<'a> fn(&'a u8)`) and `use_bounds` // (`impl Tr + use<'a, T>`) are generic-parameter lists under other -// names. `LT2` is the `token(prec(1, '<'))` spelling `type_arguments` -// and `use_bounds` open with; the parser reports it as `LT`, so it is -// listed defensively, here and in the operator arm, and its absence is -// pinned by `rust_generic_opener_alias_never_reaches_kind_id`. +// names, and a qualified path's `bracketed_type` (`::f`, +// `::f`) encloses a type (#1567). `LT2` is the +// `token(prec(1, '<'))` spelling `type_arguments` and `use_bounds` open +// with; the parser reports it as `LT`, so it is listed defensively, +// here and in the operator arm, and its absence is pinned by +// `rust_generic_opener_alias_never_reaches_kind_id`. const GENERIC_ANGLES: GenericAngleKinds = GenericAngleKinds { lists: &[ Rust::TypeArguments as u16, Rust::TypeParameters as u16, Rust::ForLifetimes as u16, Rust::UseBounds as u16, + Rust::BracketedType as u16, ], openers: &[Rust::LT as u16, Rust::LT2 as u16], closers: &[Rust::GT as u16], diff --git a/big-code-analysis-ast/src/getter/tsx.rs b/big-code-analysis-ast/src/getter/tsx.rs index d71365b8c..5ff4d9925 100644 --- a/big-code-analysis-ast/src/getter/tsx.rs +++ b/big-code-analysis-ast/src/getter/tsx.rs @@ -93,6 +93,9 @@ impl Getter for TsxCode { // `Satisfies` / `Keyof` / `Infer` / `Is` / `Asserts2`, the type-level // operator keywords: see the TS invocation (#1561). // + // `Public` through `Accessor`, the member modifiers: see the TS + // invocation (#1563). + // // `generic_angles`: see the TS invocation (#1559). A JSX tag's `<` / // `>` hang from the JSX element nodes, not from a generic list, so // they stay `<` and `>`. @@ -100,7 +103,8 @@ impl Getter for TsxCode { Tsx, op_extras: [ QMARKDOT, PredefinedType, Interface, Enum, Type, Namespace, Module2, Declare, Global, - Satisfies, Keyof, Infer, Is, Asserts2, LTSLASH, SLASHGT, + Satisfies, Keyof, Infer, Is, Asserts2, Public, Private, Protected, Override, Readonly, + Abstract, Accessor, LTSLASH, SLASHGT, ], operand_extras: [Identifier2, String2, TypeIdentifier], predefined_void: PredefinedType, diff --git a/big-code-analysis-ast/src/getter/typescript.rs b/big-code-analysis-ast/src/getter/typescript.rs index 63e458ab7..f9a2a596f 100644 --- a/big-code-analysis-ast/src/getter/typescript.rs +++ b/big-code-analysis-ast/src/getter/typescript.rs @@ -131,11 +131,30 @@ impl Getter for TypescriptCode { // type assertion wraps a `type_arguments` node, so it is a pair too; // a comparison's `<` / `>` sit under `binary_expression` and stay // their own operators. + // + // #1563 bills the member modifiers, as C# and Kotlin bill theirs: + // `public`, `private`, `protected`, `override`, `readonly`, + // `abstract` and `accessor`, beside the `static` the macro bills for + // the whole family and the `declare` above. Each is its keyword + // leaf, one kind apiece. `public` / `private` / `protected` sit + // under an `accessibility_modifier` and `override` under an + // `override_modifier`; both wrappers stay unlisted, so a modifier + // bills once. The same leaves bill outside a class too: a parameter + // property (`constructor(private x)`), `readonly T[]`, a mapped + // type's `-readonly`, `abstract class` and `abstract new () => T`. + // Used as a name (`let readonly = 1`, `o.public`, `abstract()` in an + // expression, a field named `static`) the grammar emits an + // identifier. The pinned grammar misreads a few valid name + // spellings as the keyword under or beside an ERROR — a statement + // `abstract();`, a field `abstract;` or `accessor = 1`, and an + // object-type member `abstract: T` — so those bill the operator. + // They billed nothing before, and none occurs in the corpora. impl_js_family_get_op_type!( Typescript, op_extras: [ QMARKDOT, PredefinedType, Interface, Enum, Type, Namespace, Module2, Declare, Global, - Satisfies, Keyof, Infer, Is, Asserts2, + Satisfies, Keyof, Infer, Is, Asserts2, Public, Private, Protected, Override, Readonly, + Abstract, Accessor, ], operand_extras: [TypeIdentifier], predefined_void: PredefinedType, diff --git a/big-code-analysis-ast/src/lang_helpers/elixir.rs b/big-code-analysis-ast/src/lang_helpers/elixir.rs index 5fff724aa..b6a36dc5f 100644 --- a/big-code-analysis-ast/src/lang_helpers/elixir.rs +++ b/big-code-analysis-ast/src/lang_helpers/elixir.rs @@ -98,7 +98,7 @@ pub fn elixir_do_block_call_children<'a>( /// recovery the parent can be an `ERROR` with no operands, so every /// decision metric keys on this parent rather than on the token (#1531, /// #1534). A remote call that names the operator still *applies* it, -/// though; see [`elixir_remote_call_applies`]. +/// though; see [`elixir_applying_remote_call`]. /// /// Matched by rule name rather than by `kind_id`: the grammar aliases /// `binary_operator` to three ids (`Elixir::BinaryOperator` through @@ -114,39 +114,204 @@ pub fn elixir_applying_operator<'a>( .filter(|parent| parent.kind() == "binary_operator") } -/// Whether `op`, an operator token, is the name of a remote call on -/// `Kernel` with arguments: `Kernel.||(a, b)`, or `x |> Kernel.||(y)`, -/// which the pipe rewrites to it. Such a call expands the `Kernel` macro -/// exactly as `a || b` does, so it short-circuits and decides as the -/// operator does, where a capture (`&Kernel.||/2`, a call with no -/// arguments) only names it. It has no `binary_operator`, so it carries no -/// `left` / `right` operands either. +/// The name of the short-circuit operator a remote call on `Kernel` with +/// arguments applies, or `None` for any other node: the `||` of +/// `Kernel.||(a, b)`, or of `x |> Kernel.||(y)`, which the pipe rewrites +/// to it. Such a call expands the `Kernel` macro exactly as `a || b` +/// does, so it short-circuits and decides as the operator does, where a +/// capture (`&Kernel.||/2`, a call with no arguments) only names it. It +/// has no `binary_operator`, so its operands are the call's arguments, +/// and under a pipe the piped value, rather than `left` / `right` fields. /// /// The receiver must spell `Kernel` (or `Elixir.Kernel`): a user module /// may define `||/2` as an ordinary function, and `Foo.||(a, b)` neither /// short-circuits nor decides. /// +/// The grammar lexes a symbolic name as a token inside the call's +/// `operator_identifier`, which is what is returned. `Kernel.and(a, b)` +/// and `Kernel.or(a, b)` spell their names as a plain `identifier`, with +/// no `And` / `Or` token to find, so that identifier is returned instead +/// (#1582). Any other name — `Kernel.+`, `Kernel.!`, `Kernel.not`, +/// `Kernel.in` — is a function or decides nothing, and is not found; nor +/// is the childless `..`. [`elixir_short_circuit_name`] says which +/// operation the name applies. +/// +/// Every identifier a walk meets asks this of its call, so the receiver, +/// which turns nearly all of them away without a cursor, is read before +/// the call's children are scanned for its `arguments`. +/// /// Matched by rule name: `dot` and `arguments` are aliased to several /// ids each (grammar-dispatch §1). #[must_use] -pub fn elixir_remote_call_applies<'a>( +pub fn elixir_kernel_short_circuit_name<'a>(call: &Node<'a>, code: &[u8]) -> Option> { + kernel_short_circuit_parts(call, code).map(|(name, _)| name) +} + +fn kernel_short_circuit_parts<'a>(call: &Node<'a>, code: &[u8]) -> Option<(Node<'a>, Node<'a>)> { + if call.kind() != "call" { + return None; + } + let dot = call.child_by_field_name("target").filter(|target| { + target.kind() == "dot" + && target.child_by_field_name("left").is_some_and(|receiver| { + matches!( + code.get(receiver.start_byte()..receiver.end_byte()), + Some(b"Kernel" | b"Elixir.Kernel") + ) + }) + })?; + let name = dot.child_by_field_name("right")?; + let name = if name.kind() == "operator_identifier" { + name.child(0)? + } else { + name + }; + elixir_short_circuit_name(&name, code) + .and_then(|_| call.children().find(|child| child.kind() == "arguments")) + .map(|arguments| (name, arguments)) +} + +/// The short-circuit operation an operator token kind applies, keyed by +/// its symbol: `&&` for `&&` and `and`, `||` for `||` and `or`. A word +/// form demands a boolean left operand, but it is the same conjunction or +/// disjunction to read, so one sequence key serves both spellings. +/// `None` for any other kind. +#[must_use] +pub fn elixir_short_circuit_kind(kind_id: u16) -> Option { + match kind_id.into() { + Elixir::AMPAMP | Elixir::And => Some(Elixir::AMPAMP), + Elixir::PIPEPIPE | Elixir::Or => Some(Elixir::PIPEPIPE), + _ => None, + } +} + +/// [`elixir_short_circuit_kind`] for an operator's name as +/// [`elixir_kernel_short_circuit_name`] returns it: a token, or the +/// `identifier` spelling `Kernel.and` / `Kernel.or` (#1582), whose +/// bytes decide. Ask it only of such a name — an identifier is an `and` +/// operator nowhere else. +#[must_use] +pub fn elixir_short_circuit_name(name: &Node, code: &[u8]) -> Option { + if name.kind_id() != Elixir::Identifier as u16 { + return elixir_short_circuit_kind(name.kind_id()); + } + match code.get(name.start_byte()..name.end_byte())? { + b"and" => Some(Elixir::AMPAMP), + b"or" => Some(Elixir::PIPEPIPE), + _ => None, + } +} + +/// A recognized short-circuit `Kernel` call and the context its operands need. +pub struct ElixirRemoteCall<'tree, 'chain> { + /// The call that applies the operator. + pub call: Node<'tree>, + /// The argument list found while recognizing the call. + pub arguments: Node<'tree>, + /// The call's own ancestry. + pub ancestors: Ancestors<'tree, 'chain>, +} + +/// The `Kernel` remote call that applies `op`, a short-circuit operator's +/// name — the call [`elixir_kernel_short_circuit_name`] answers `op` for +/// — with its argument list and ancestry, or `None` when `op` sits +/// anywhere else. A token's parents are the `operator_identifier`, then +/// the `dot`, then the call; the `identifier` naming `Kernel.and` sits +/// on the `dot` directly. Any other node fails on the first one or two +/// parents' kinds, so the check is cheap enough to ask of every +/// identifier a walk meets. +#[must_use] +pub fn elixir_applying_remote_call<'a, 'c>( op: &Node<'a>, code: &[u8], - ancestors: Ancestors<'a, '_>, -) -> bool { - let mut up = ancestors.iter(op).map(|(node, _)| node); - up.next() - .is_some_and(|name| name.kind() == "operator_identifier") - && up.next().is_some_and(|dot| { - dot.kind() == "dot" - && dot.child_by_field_name("left").is_some_and(|receiver| { - matches!( - code.get(receiver.start_byte()..receiver.end_byte()), - Some(b"Kernel" | b"Elixir.Kernel") - ) - }) - }) - && up.next().is_some_and(|call| { - call.kind() == "call" && call.children().any(|child| child.kind() == "arguments") + ancestors: Ancestors<'a, 'c>, +) -> Option> { + let mut up = ancestors.iter(op); + let (mut dot, _) = up.next()?; + if dot.kind() == "operator_identifier" { + dot = up.next()?.0; + } + if dot.kind() != "dot" { + return None; + } + let (call, call_ancestors) = up.next()?; + kernel_short_circuit_parts(&call, code) + .filter(|(name, _)| name.id() == op.id()) + .map(|(_, arguments)| ElixirRemoteCall { + call, + arguments, + ancestors: call_ancestors, }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ElixirCode; + use crate::node::Tree; + use crate::test_support::for_each_node_with_chain; + + #[cfg(feature = "elixir")] + #[test] + fn short_circuit_name_requires_the_identifiers_source_span() { + let source = b"Kernel.and(a, b)"; + let tree = Tree::new::(source); + let name = tree + .get_root() + .preorder() + .find(|node| { + node.kind_id() == Elixir::Identifier as u16 && node.utf8_text(source) == Some("and") + }) + .expect("fixture contains the operator name"); + assert_eq!( + elixir_short_circuit_name(&name, source), + Some(Elixir::AMPAMP) + ); + assert_eq!( + elixir_short_circuit_name(&name, &source[..name.end_byte() - 1]), + None + ); + } + + #[cfg(feature = "elixir")] + #[test] + fn remote_call_lookup_rejects_a_root_and_keeps_its_argument_list() { + let source = b"Kernel.and(a, b)\nKernel.||(c, d)\n"; + let mut recognized = Vec::new(); + for_each_node_with_chain::(source, |node, chain| { + let known = elixir_applying_remote_call(node, source, Ancestors::known(chain)); + let unknown = elixir_applying_remote_call(node, source, Ancestors::unknown()); + assert_eq!( + known.as_ref().map(|r| (r.call.id(), r.arguments.id())), + unknown.map(|r| (r.call.id(), r.arguments.id())) + ); + if chain.is_empty() { + assert!(known.is_none()); + } + if let Some(remote) = known { + assert_eq!(remote.arguments.kind(), "arguments"); + assert_eq!( + remote.ancestors.parent(&remote.call).map(|p| p.kind()), + Some("source") + ); + recognized.push(( + node.utf8_text(source) + .expect("operator is UTF-8") + .to_owned(), + remote + .arguments + .utf8_text(source) + .expect("arguments are UTF-8") + .to_owned(), + )); + } + }); + assert_eq!( + recognized, + [ + ("and".to_owned(), "(a, b)".to_owned()), + ("||".to_owned(), "(c, d)".to_owned()) + ] + ); + } +} diff --git a/big-code-analysis-ast/src/lang_helpers/perl.rs b/big-code-analysis-ast/src/lang_helpers/perl.rs index de72ed6c1..1e4b4eec2 100644 --- a/big-code-analysis-ast/src/lang_helpers/perl.rs +++ b/big-code-analysis-ast/src/lang_helpers/perl.rs @@ -1,5 +1,6 @@ -//! Perl: telling the `and` and `not` operators, and the file tests, -//! from an auto-quoted key that the grammar lexes as one of them. +//! Perl: telling the `and` and `not` operators, the file tests, the +//! unary minus and bareword calls from an auto-quoted key that the +//! grammar lexes as one of them. use crate::Perl; use crate::node::{Ancestors, Node}; @@ -114,10 +115,9 @@ pub fn perl_not_is_key<'a>(not: &Node<'a>, parent: &Node<'a>) -> bool { /// The subscript form of the second shape (`$h{-x}`) recovers into an /// `ERROR` that drops the `-x` token altogether, so nothing is left to /// bill. A key whose letter is no file-test letter (`-name`, `-height`) -/// is a third shape, a `-` applied to the bareword, which this does not -/// cover: it still bills the `-` and the word, and ABC counts the word -/// as a call, as every bareword key does. Remove this, and every call -/// site, once the pin carries the upstream fix. +/// is a third shape, a `-` applied to the bareword, which +/// [`perl_bareword_is_key`] covers. Remove this, and every call site, +/// once the pin carries the upstream fix. #[must_use] pub(crate) fn perl_file_test_is_dash_key<'a>( test: &Node<'a>, @@ -130,7 +130,12 @@ pub(crate) fn perl_file_test_is_dash_key<'a>( if operand.is_error() { operand.is_child(Perl::FatComma as u16) && operand.start_row() == test.start_row() } else { - is_glued_word(&operand, test) && is_key_position(test, ancestors) + is_glued_word(&operand, test) + && is_plain_bareword(&operand) + && ancestors + .iter(test) + .next() + .is_some_and(|(key, above)| is_key_position(&key, above)) } }) } @@ -139,7 +144,7 @@ pub(crate) fn perl_file_test_is_dash_key<'a>( /// file test bills whole, so it bills nothing of its own. `ancestors` /// is `node`'s chain. #[must_use] -pub fn perl_is_dash_key_word<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { +pub(crate) fn perl_is_dash_key_word<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { ancestors.iter(node).next().is_some_and(|(test, above)| { test.kind_id() == Perl::FileHandleOperator as u16 && is_glued_word(node, &test) @@ -147,25 +152,127 @@ pub fn perl_is_dash_key_word<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) }) } -/// Whether `dash`, a Perl `-` token whose parent is `parent`, is the -/// sign of an auto-quoted `-and` or `-not` key (`$h{-not}`, -/// `(-and => 1)`), which the grammar splits into the `-` and the keyword -/// token. [`perl_and_is_operator`] and [`perl_not_is_key`] decide -/// whether the keyword is a key, and a key's sign is part of the string -/// whether or not a space separates the two. +/// Whether `word`, a Perl `call_expression_with_bareword` whose +/// ancestor chain is `ancestors`, is an auto-quoted hash key rather than +/// a call. +/// +/// FIXME(#1571 upstream): tree-sitter-perl 1.1.2 parses every bareword +/// key as a call. Perl quotes a word of letters, digits and underscores +/// before `=>` (perlop, "Comma Operator") and alone in a hash subscript +/// (perldata, "Scalar value constructors"), a leading `-` included, so +/// `(name => 1)`, `(-name => 1)`, `$h{-name}` and `$h{-size}` hold the +/// strings `"name"`, `"-name"`, `"-name"` and `"-size"`, and call +/// nothing. Three shapes reach here: +/// +/// - the bareword itself before a `=>` (`$h{name}` is an `identifier` +/// with no call around it, so the subscript form never arises); +/// - the operand of a `-` whose `unary_expression` is the key — +/// before a `=>`, or alone in a subscript; +/// - the word of a `-bareword` key the grammar reads as a file test +/// (`-size`), which `perl_is_dash_key_word` decides. +/// +/// A package-qualified word (`Foo::Bar => 1`) is not quoted by `=>` +/// and stays a call, as does a word with arguments (`name() => 1`), +/// which the grammar wraps in a call of its own. Remove this, and every +/// call site, once the pin carries the upstream fix. +#[must_use] +pub fn perl_bareword_is_key<'a>(word: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + if perl_is_dash_key_word(word, ancestors) { + return true; + } + if !is_plain_bareword(word) { + return false; + } + match ancestors.iter(word).next() { + Some((unary, above)) if dash_operator(&unary).is_some() => is_key_position(&unary, above), + _ => is_key_position(word, ancestors), + } +} + +/// Whether `word`, a Perl `call_expression_with_bareword` whose ancestor +/// chain is `ancestors`, is a call site of its own. +/// +/// The grammar nests the callee of every call that takes arguments as a +/// bareword call: `foo(1)`, `foo 1` and `sort { … } @x` each wrap a +/// `call_expression_with_bareword` spelling `foo` or `sort`, so the +/// wrapper is the call and the word inside it is its name. The word is +/// kept for the spellings with no wrapper — `foo;`, `&foo;`, and the +/// `foo(1` that error recovery leaves without one — so the wrapper +/// cannot be dropped instead. Nor is an auto-quoted key a call +/// ([`perl_bareword_is_key`]). ABC's branches and +/// `Checker::is_call_with_code` both ask this, so the two count the same +/// calls. +#[must_use] +pub fn perl_bareword_is_call<'a>(word: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + !ancestors.parent(word).is_some_and(|parent| { + matches!( + parent.kind_id().into(), + Perl::CallExpressionWithSpacedArgs + | Perl::CallExpressionWithSub + | Perl::CallExpressionWithArgsWithBrackets + | Perl::CallExpressionWithVariable + | Perl::CallExpressionRecursive + ) + }) && !perl_bareword_is_key(word, ancestors) +} + +/// Whether `dash`, a Perl `-` token whose ancestor chain is +/// `ancestors`, is the sign of an auto-quoted key rather than a unary +/// minus: a `-and` / `-not` key ([`perl_dash_signs_keyword_key`]) or a +/// `-bareword` key ([`perl_bareword_is_key`]). A key's sign is part of +/// the string, so it is no operator. +#[must_use] +pub(crate) fn perl_dash_signs_key<'a>(dash: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + perl_dash_signs_keyword_key(dash, ancestors) + || ancestors.iter(dash).next().is_some_and(|(parent, above)| { + parent.kind_id() == Perl::UnaryExpression as u16 + && next_non_extra(dash, ancestors).is_some_and(|word| is_plain_bareword(&word)) + && is_key_position(&parent, above) + }) +} + +/// The `-` that signs the `-bareword` key `identifier` spells, or +/// `None` when `identifier`, a Perl `identifier` whose ancestor chain is +/// `ancestors`, is no such key's word. The key bills as one operand +/// spelled from the `-` to the word's end, so `(-name => 1)` bills +/// `-name`, the spelling of its quoted twin's contents. +#[must_use] +pub(crate) fn perl_bareword_key_sign<'a>( + identifier: &Node<'a>, + ancestors: Ancestors<'a, '_>, +) -> Option> { + let mut up = ancestors.iter(identifier); + let (_, word_ancestors) = up + .next() + .filter(|(word, _)| word.kind_id() == Perl::CallExpressionWithBareword as u16)?; + let (unary, _) = up.next()?; + let dash = dash_operator(&unary)?; + // `word` and `dash` are siblings, so they share an ancestry. + perl_dash_signs_key(&dash, word_ancestors).then_some(dash) +} + +/// Whether `dash`, a Perl `-` token whose ancestor chain is +/// `ancestors`, is the sign of an auto-quoted `-and` or `-not` key +/// (`$h{-not}`, `(-and => 1)`), which the grammar splits into the `-` +/// and the keyword token. [`perl_and_is_operator`] and +/// [`perl_not_is_key`] decide whether the keyword is a key, and a key's +/// sign is part of the string whether or not a space separates the two. #[must_use] -pub fn perl_dash_signs_keyword_key<'a>(dash: &Node<'a>, parent: &Node<'a>) -> bool { +pub fn perl_dash_signs_keyword_key<'a>(dash: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { // A sign is a prefix `-`, whose parent is the subscript's `ERROR` or // a `unary_expression`. An infix `-` is a real subtraction even // before an auto-quoted key (`(1 - and => 2)` subtracts the string), // and is by far the commonest `-`, so it is answered first. - if parent.kind_id() == Perl::BinaryExpression as u16 { + let Some(parent) = ancestors + .parent(dash) + .filter(|parent| parent.kind_id() != Perl::BinaryExpression as u16) + else { return false; - } + }; // `$h{-not}` keeps the keyword beside the `-`; `(-not => 1)` nests // it as the first child of the `-`'s operand. - next_non_extra(parent, dash).is_some_and(|next| { - keyword_is_key(&next, parent) + next_non_extra(dash, ancestors).is_some_and(|next| { + keyword_is_key(&next, &parent) || first_non_extra(&next).is_some_and(|keyword| keyword_is_key(&keyword, &next)) }) } @@ -179,18 +286,19 @@ pub fn perl_dash_signs_keyword_key<'a>(dash: &Node<'a>, parent: &Node<'a>) -> bo /// sign and its key. #[must_use] pub fn perl_key_is_signed<'a>(key: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - let mut up = ancestors.iter(key).map(|(ancestor, _)| ancestor); - up.next().is_some_and(|parent| { - let beside = parent - .children_before(key) + let mut up = ancestors.iter(key); + up.next().is_some_and(|(_, parent_ancestors)| { + let beside = ancestors + .siblings_before(key) .find(|sibling| !sibling.as_tree_sitter().is_extra()); + // A `-` beside the key shares its chain; the `operator` of the + // expression the key leads is a sibling of that expression. if let Some(dash) = beside.filter(|node| node.kind_id() == Perl::DASH as u16) { - return perl_dash_signs_keyword_key(&dash, &parent); + return perl_dash_signs_keyword_key(&dash, ancestors); } - up.next().is_some_and(|outer| { - outer.child_by_field_name("operator").is_some_and(|dash| { - dash.kind_id() == Perl::DASH as u16 && perl_dash_signs_keyword_key(&dash, &outer) - }) + up.next().is_some_and(|(outer, _)| { + dash_operator(&outer) + .is_some_and(|dash| perl_dash_signs_keyword_key(&dash, parent_ancestors)) }) }) } @@ -215,26 +323,83 @@ fn is_glued_word(word: &Node, test: &Node) -> bool { && word.start_byte() == test.start_byte() + FILE_TEST_TOKEN_LEN } -// Whether the `unary_expression` around `test` is the whole key of a -// hash subscript, which the grammar wraps in a `binary_expression` with -// no other child but comments, or is followed by a `=>`. -fn is_key_position<'a>(test: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { - let mut up = ancestors.iter(test).map(|(ancestor, _)| ancestor); - let mut key = up.next(); - let mut parent = up.next(); - if parent - .is_some_and(|p| p.kind_id() == Perl::BinaryExpression as u16 && non_extra_count(&p) == 1) - { - key = parent; - parent = up.next(); +// Whether `key`, whose ancestor chain is `ancestors`, is a key where +// Perl auto-quotes one: the whole of a hash subscript, which the grammar +// may wrap in a `binary_expression` with no other child but comments, +// or the last thing before a `=>`. Every auto-quoted key shape asks this +// one question, of the node that spans its whole key. +// +// `=>` quotes the word immediately to its left whatever expression that +// word closes (perlop, "Comma Operator"), so `(1 - name => 2)` subtracts +// the string `"name"`. The grammar nests such a word as the last operand +// of its expression, so the test climbs while the node is its parent's +// last child: the sibling after the first ancestor that has one is the +// token after the word. A key that stopped being the whole of its node +// on the way up is no subscript key (`$h{1 - name}` is not quoted). +fn is_key_position<'a>(key: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { + let (mut node, mut node_ancestors) = (*key, ancestors); + let mut whole = true; + for (parent, above) in ancestors.iter(key) { + if whole + && matches!( + parent.kind_id().into(), + Perl::HashAccessVariableSimple | Perl::HashAccessVariable + ) + { + return true; + } + // As in `perl_not_is_key`, only comments are passed over: the + // grammar wraps a stray `,` (`(name, => 1)`) in an `ERROR` it + // marks as an extra, and that `,`, not the word, is then the + // token the `=>` quotes, which is nothing. A `-foo, =>` file-test + // key stops here too and reads as the grammar does (a file test + // on `oo`), as every `-bareword` outside key position does; Perl + // itself reads one as a string or a negated call by whether a sub + // of that name is visible, which no single file says. + if let Some(next) = node_ancestors + .siblings_after(&node) + .find(|sibling| sibling.is_error() || !sibling.as_tree_sitter().is_extra()) + { + return next.kind_id() == Perl::FatComma as u16; + } + whole &= parent.kind_id() == Perl::BinaryExpression as u16 && non_extra_count(&parent) == 1; + (node, node_ancestors) = (parent, above); } - key.zip(parent).is_some_and(|(key, parent)| { - matches!( - parent.kind_id().into(), - Perl::HashAccessVariableSimple | Perl::HashAccessVariable - ) || next_non_extra(&parent, &key) - .is_some_and(|next| next.kind_id() == Perl::FatComma as u16) - }) + false +} + +// Whether `word` is a call of a bare word and nothing else: one +// `identifier`, no package qualifier, no arguments. Only such a word is +// auto-quoted. +fn is_plain_bareword(word: &Node) -> bool { + if word.kind_id() != Perl::CallExpressionWithBareword as u16 { + return false; + } + // Every call's name is one of these, so the common shape — one + // child, no comment inside the call — is read off two fields rather + // than through a cursor over the children. + match word.child_count() { + 1 => word + .child(0) + .is_some_and(|child| child.kind_id() == Perl::Identifier as u16), + _ => word + .children() + .filter(|child| !child.as_tree_sitter().is_extra()) + .map(|child| child.kind_id()) + .eq([Perl::Identifier as u16]), + } +} + +// The `-` of `node` when it is a unary minus — the `unary_expression` a +// `-bareword` key whose letter is no file-test letter parses as — and +// `None` for any other node, a subtraction's `binary_expression` +// included. +fn dash_operator<'a>(node: &Node<'a>) -> Option> { + if node.kind_id() != Perl::UnaryExpression as u16 { + return None; + } + node.child_by_field_name("operator") + .filter(|op| op.kind_id() == Perl::DASH as u16) } fn non_extra_count(node: &Node) -> usize { @@ -243,12 +408,13 @@ fn non_extra_count(node: &Node) -> usize { .count() } -// The sibling after `child` in `parent`, comments aside. Every key of -// a wide `(-a => 1, -b => 2, …)` list asks, so it is found without a -// scan from the list's start (#1545). -fn next_non_extra<'a>(parent: &Node<'a>, child: &Node<'a>) -> Option> { - parent - .children_after(child) +// The sibling after `node`, whose ancestor chain is `ancestors`, +// comments aside. Every key of a wide `(-a => 1, -b => 2, …)` list or +// a flat `ERROR` asks, so it is found without a scan from the parent's +// start (#1545, #1584). +fn next_non_extra<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> Option> { + ancestors + .siblings_after(node) .find(|sibling| !sibling.as_tree_sitter().is_extra()) } diff --git a/big-code-analysis-ast/src/lib.rs b/big-code-analysis-ast/src/lib.rs index 966a6e025..39c395af3 100644 --- a/big-code-analysis-ast/src/lib.rs +++ b/big-code-analysis-ast/src/lib.rs @@ -137,7 +137,7 @@ pub use crate::getter::Getter; pub use crate::langs::*; pub use crate::languages::*; pub use crate::macros::ParseLangError; -pub use crate::node::{Ancestors, Node}; +pub use crate::node::{Ancestors, Node, SiblingCache}; pub use crate::parser::Parser; pub use crate::preproc::{ PreprocDiagnostic, PreprocFile, PreprocResults, fix_includes, get_macros, preprocess, diff --git a/big-code-analysis-ast/src/node.rs b/big-code-analysis-ast/src/node.rs index f7ce060b8..669bc3f17 100644 --- a/big-code-analysis-ast/src/node.rs +++ b/big-code-analysis-ast/src/node.rs @@ -12,8 +12,13 @@ //! Tree-sitter wrappers: `Tree`, [`Node`], [`Ancestors`], and the cursors. +mod ancestors; mod parser_cache; +#[doc(hidden)] +pub use ancestors::WIDE_PARENT; +pub use ancestors::{AncestorIter, Ancestors, SiblingCache}; + use tree_sitter::Node as OtherNode; use tree_sitter::Tree as OtherTree; use tree_sitter::TreeCursor; @@ -511,8 +516,12 @@ impl<'a> Node<'a> { /// and nothing when `child` is not one of this node's children. /// /// That scan costs `child`'s index, so a walk that asks it of every - /// child of a wide list is quadratic in the list (#1545). This finds - /// `child` in `O(log n)` instead; see [`Node::children_before`]. + /// child of a wide list is quadratic in the list (#1545). This seeks + /// `child` by byte instead, which is `O(log n)` in a list the grammar + /// stores as a `repeat()` and still `O(index)` in a flat one, an + /// `ERROR` above all; see [`Cursor::goto_first_child_for_byte`]. A + /// walk that asks of every child of a node that may be flat goes + /// through [`Ancestors::siblings_after`] (#1584). pub fn children_after(&self, child: &Node<'a>) -> impl Iterator> + use<'a> { let mut cursor = self.cursor(); let mut more = @@ -529,8 +538,11 @@ impl<'a> Node<'a> { /// /// `child`'s index comes from [`Cursor::goto_first_child_for_byte`] /// and each sibling from [`Node::child`], both `O(log n)` in the - /// child count, for the reason the first gives: a `repeat()` list is - /// stored as a balanced tree of hidden nodes. + /// child count where the list is a `repeat()`, stored as a balanced + /// tree of hidden nodes, and both `O(index)` where it is flat, as an + /// `ERROR` built by skipping tokens is. A walk that asks of every + /// child of such a node goes through [`Ancestors::siblings_before`] + /// (#1584). pub fn children_before(&self, child: &Node<'a>) -> impl Iterator> + use<'a> { let node = *self; let index = self.seat_on_child(child, &mut self.cursor()).unwrap_or(0); @@ -663,212 +675,6 @@ impl<'a> Node<'a> { } } -/// The chain of a node's ancestors, root first, as recorded by a walker -/// that descended to that node. -/// -/// A predicate that asks a node for its parent pays [`Node::parent`]'s -/// `O(depth)` once per node, so `O(depth²)` over a deeply nested file, -/// however few parent steps it takes (#1084). A walker that visits -/// parents before children already holds the chain, and handing it down -/// turns each step into a slice index — the upward counterpart of the -/// downward flag propagation #1052 and #1062 used. -/// -/// Callers that reached a node some other way pass -/// [`Ancestors::unknown`], which climbs with [`Node::parent`]: the same -/// answers at the original cost. -#[derive(Clone, Copy, Debug)] -pub struct Ancestors<'tree, 'chain>(Option<&'chain [Node<'tree>]>); - -impl<'tree, 'chain> Ancestors<'tree, 'chain> { - /// No chain is available; every query climbs with [`Node::parent`]. - #[inline] - #[must_use] - pub const fn unknown() -> Self { - Self(None) - } - - /// `chain` lists every ancestor of the node about to be queried, - /// root first — so `chain.last()` is its parent and an empty chain - /// means the node is the root. - #[inline] - #[must_use] - pub const fn known(chain: &'chain [Node<'tree>]) -> Self { - Self(Some(chain)) - } - - /// [`Ancestors::known`], but first checks that `chain` really is - /// `node`'s ancestry. - /// - /// The parity test below proves the walker's truncate/push rule on a - /// *replica* walker, so it cannot see `metrics_inner` itself - /// desynchronising — a `chain.push` moved ahead of the per-node - /// computes, say, or a `continue` inserted above the truncate. - /// [`Ancestors::parent`] trusts `chain.last()` unvalidated, so such a - /// drift would feed every predicate a wrong ancestor silently rather - /// than fail. Walkers that maintain a chain should construct through - /// here; [`Ancestors::known`] stays unchecked for the callers that - /// deliberately pair a chain with a foreign node. - /// - /// # Two checks, because the exact one is not affordable by default - /// - /// The invariant is "`chain.last()` **is** `node.parent()`", and - /// asking that outright costs [`Node::parent`]'s `O(depth)` — the - /// very lookup #1084 exists to remove. Per node, on every walk that - /// constructs a checked chain, it made every debug-build walk - /// `O(nodes × depth)` while the shipped walk is `O(nodes)`: a tax on - /// every `cargo test`, worst on the deep-nesting regression tests - /// that exist to pin the shipped walk's linearity (#1122). It now - /// runs only under `--cfg chain_audit`, which `make chain-audit` and - /// the CI lane of the same name set — and there as a plain - /// `assert_eq!`, so the audit has teeth in a release profile too. - /// - /// What stays on in every debug build is an `O(1)` *consequence* of - /// that invariant: a parent's byte span contains its child's, and no - /// node is its own parent. Strictly weaker — a grandparent contains - /// the node as well — but it is shaped to the two ways a walker - /// desynchronises. A `push` moved ahead of the per-node computes - /// leaves `chain.last() == node`; a dropped `truncate` leaves the - /// previous subtree's path, whose last entry is disjoint from the - /// node that follows it. Both trip here, on the first node that - /// shows them, at four integer comparisons. - #[must_use] - pub fn checked(chain: &'chain [Node<'tree>], node: &Node<'tree>) -> Self { - // Opt-in only: `Node::parent` restarts at the root, so this is - // the `O(nodes × depth)` walk described above. - #[cfg(chain_audit)] - assert_eq!( - chain.last().map(Node::id), - node.parent().map(|parent| parent.id()), - "ancestor chain desynchronised on a {} node", - node.kind() - ); - debug_assert!( - chain.last().is_none_or(|parent| { - parent.id() != node.id() - && parent.start_byte() <= node.start_byte() - && node.end_byte() <= parent.end_byte() - }), - "ancestor chain desynchronised on a {} node: chain.last() neither \ - contains it nor differs from it", - node.kind() - ); - Self::known(chain) - } - - /// How far the node this chain describes sits from the root, or - /// `None` when no chain is known — deriving it then would cost the - /// [`Node::parent`] climb the chain exists to remove. - #[inline] - pub fn depth(self) -> Option { - self.0.map(<[Node<'tree>]>::len) - } - - /// `node`'s parent. - #[inline] - #[must_use] - pub fn parent(self, node: &Node<'tree>) -> Option> { - match self.0 { - Some(chain) => chain.last().copied(), - None => node.parent(), - } - } - - /// Whether `node`'s parent has kind id `kind`. - /// - /// The compound-leaf guard of `.claude/rules/grammar-dispatch.md` - /// section 5 asks this and nothing else: a delimiter or keyword - /// token is suppressed *only* directly under the construct that - /// owns it, never under an arbitrary ancestor. Eleven dispatch - /// arms across `getter`, `checker` and the metric walkers spelled - /// it out as `.parent(node).is_some_and(|p| p.kind_id() == X as - /// u16)`, which wraps onto four rustfmt lines inside a match guard - /// and buries the question under the plumbing; #1314 added six more - /// and folded all seventeen onto this. Three sites spelling their - /// binding `|parent|` were left alone and would each fit: - /// `impl_is_else_if_parent_clause!` (`src/checker.rs`), - /// `is_stabby_lambda_body` (`src/checker/ruby.rs`) and - /// `is_useful_comment` (`src/checker/rust.rs`, a let-chain). - /// - /// A `false` when `node` has no parent is the answer every one of - /// those call sites wants: a root node's token is not inside the - /// construct, so it is not suppressed. - #[inline] - #[must_use] - pub fn parent_has_kind(self, node: &Node<'tree>, kind: u16) -> bool { - self.parent(node).is_some_and(|p| p.kind_id() == kind) - } - - /// The sibling immediately before `node`, or `None` when `node` is - /// its parent's first child or has no parent. - /// - /// `tree_sitter`'s own `prev_sibling` resolves the parent first - /// (`ts_node__prev_sibling` opens with `ts_node_parent`), so it - /// carries the same `O(depth)` cost [`Node::parent`] does. With a - /// known chain the parent is free and what remains is a cursor walk - /// over the siblings. - #[inline] - #[must_use] - pub fn previous_sibling(self, node: &Node<'tree>) -> Option> { - let Some(chain) = self.0 else { - return node.previous_sibling(); - }; - let parent = chain.last()?; - let mut previous = None; - for child in parent.children() { - if child.id() == node.id() { - return previous; - } - previous = Some(child); - } - // `node` is not among `parent.children()`, so this chain does - // not describe `node`. Answering `None` would claim "no - // previous sibling", which is a different (and wrong) answer; - // fall back to the authoritative lookup instead. - node.previous_sibling() - } - - /// `node`'s ancestors, nearest first, each paired with *its* own - /// ancestry so a predicate applied to an ancestor stays as cheap as - /// one applied to `node`. - #[inline] - #[must_use] - pub fn iter(self, node: &Node<'tree>) -> AncestorIter<'tree, 'chain> { - match self.0 { - Some(chain) => AncestorIter::Chain(chain), - None => AncestorIter::Climb(node.parent()), - } - } -} - -/// Ancestor iterator returned by [`Ancestors::iter`], nearest first. -pub enum AncestorIter<'tree, 'chain> { - /// The not-yet-yielded prefix of a known chain. Its last element is - /// the next ancestor, and the prefix before it is that ancestor's - /// own chain — so splitting from the back hands out both at once. - Chain(&'chain [Node<'tree>]), - /// The next ancestor to yield, reached by climbing. - Climb(Option>), -} - -impl<'tree, 'chain> Iterator for AncestorIter<'tree, 'chain> { - type Item = (Node<'tree>, Ancestors<'tree, 'chain>); - - fn next(&mut self) -> Option { - match self { - Self::Chain(remaining) => { - let (&nearest, above) = remaining.split_last()?; - *remaining = above; - Some((nearest, Ancestors::known(above))) - } - Self::Climb(next) => { - let nearest = (*next)?; - *next = nearest.parent(); - Some((nearest, Ancestors::unknown())) - } - } - } -} - /// Pre-order iterator over a node and its descendants, returned by /// [`Node::preorder`]. /// @@ -942,6 +748,11 @@ impl<'a> Cursor<'a> { /// is `O(log n)` in the sibling count where `children().position(..)` /// is `O(n)` — the difference between a linear and a quadratic walk /// when every child of a wide list asks for its own index. + /// + /// Only for such a list. The seek steps through the children in + /// order and can skip only a *hidden* subtree, so over a flat child + /// array — an `ERROR` that recovery built by skipping tokens has no + /// hidden levels — it is the same `O(index)` scan (#1584). #[inline] pub fn goto_first_child_for_byte(&mut self, byte: usize) -> Option { self.0.goto_first_child_for_byte(byte) @@ -1103,12 +914,25 @@ impl<'a> Search<'a> for Node<'a> { let mut chain: Vec> = std::iter::successors(self.parent(), Node::parent).collect(); chain.reverse(); let depth = chain.len(); + // As in the metric walk: the children of each wide node on + // `chain`, so a sibling lookup in a flat `ERROR` is a binary + // search rather than a seek from its start (#1584). Perl's + // `call` filter reads the token after every bareword (#1580). + // The seeded ancestors are recorded too, since the walk never + // pushes them. + let mut siblings = SiblingCache::default(); + for (at, ancestor) in chain.iter().enumerate() { + siblings.record(ancestor, at, ancestor.children()); + } stack.push((*self, depth)); while let Some((node, depth)) = stack.pop() { chain.truncate(depth); - action(&node, Ancestors::checked(&chain, &node)); + action( + &node, + Ancestors::checked(&chain, &node).with_siblings(&siblings), + ); chain.push(node); // Source order in, tail reversed in place, so the LIFO // `stack` yields the leftmost child first — pre-order with @@ -1118,6 +942,11 @@ impl<'a> Search<'a> for Node<'a> { node.children_with(&mut cursor) .map(|child| (child, depth + 1)), ); + siblings.record( + &node, + depth, + stack[first_child..].iter().map(|(child, _)| *child), + ); stack[first_child..].reverse(); } } @@ -2051,6 +1880,32 @@ mod tests { assert!(visited > 3, "subtree is too small to prove much"); } + /// A walk started below the root records its seeded ancestors' + /// children too, so a sibling lookup off a wide one is answered from + /// the cache. Without them the lookup's debug assertion fails: the + /// cache is lent, and the wide parent on the chain was never pushed + /// by this walk (#1580). + #[cfg(feature = "mozjs")] + #[test] + fn act_on_node_records_a_wide_seeded_ancestor() { + // Discriminates only through `Ancestors::lookup`'s `debug_assert!`: + // meaningful under `make pre-commit`'s debug build, vacuous under + // `--release` (lesson #80). + let code = "var a = 1;\n".repeat(WIDE_PARENT + 6); + let tree = Tree::new::(code.as_bytes()); + let root = tree.get_root(); + assert!(root.child_count() > WIDE_PARENT, "the root must be wide"); + let subtree = root.child(3).expect("fixture has a fourth statement"); + + let mut after = None; + subtree.act_on_node(&mut |node, ancestors| { + if node.id() == subtree.id() { + after = Some(ancestors.siblings_after(node).count()); + } + }); + assert_eq!(after, Some(root.child_count() - 4)); + } + /// `previous_sibling` must not answer "no previous sibling" when the /// chain it was handed belongs to a different node. /// diff --git a/big-code-analysis-ast/src/node/ancestors.rs b/big-code-analysis-ast/src/node/ancestors.rs new file mode 100644 index 000000000..9ce6d95b9 --- /dev/null +++ b/big-code-analysis-ast/src/node/ancestors.rs @@ -0,0 +1,554 @@ +//! [`Ancestors`], the chain of a node's ancestors a walker threads down +//! to its predicates, and the [`SiblingCache`] it can lend with it. + +use super::Node; + +/// The chain of a node's ancestors, root first, as recorded by a walker +/// that descended to that node. +/// +/// A predicate that asks a node for its parent pays [`Node::parent`]'s +/// `O(depth)` once per node, so `O(depth²)` over a deeply nested file, +/// however few parent steps it takes (#1084). A walker that visits +/// parents before children already holds the chain, and handing it down +/// turns each step into a slice index — the upward counterpart of the +/// downward flag propagation #1052 and #1062 used. +/// +/// Callers that reached a node some other way pass +/// [`Ancestors::unknown`], which climbs with [`Node::parent`]: the same +/// answers at the original cost. +#[derive(Clone, Copy, Debug)] +pub struct Ancestors<'tree, 'chain> { + chain: Option<&'chain [Node<'tree>]>, + siblings: Option<&'chain SiblingCache<'tree>>, +} + +impl<'tree, 'chain> Ancestors<'tree, 'chain> { + /// No chain is available; every query climbs with [`Node::parent`]. + #[inline] + #[must_use] + pub const fn unknown() -> Self { + Self { + chain: None, + siblings: None, + } + } + + /// `chain` lists every ancestor of the node about to be queried, + /// root first — so `chain.last()` is its parent and an empty chain + /// means the node is the root. + #[inline] + #[must_use] + pub const fn known(chain: &'chain [Node<'tree>]) -> Self { + Self { + chain: Some(chain), + siblings: None, + } + } + + /// This chain, answering [`siblings_after`](Self::siblings_after) and + /// [`siblings_before`](Self::siblings_before) through `cache`, which + /// the walker that keeps the chain owns for the whole walk. + #[inline] + #[must_use] + pub const fn with_siblings(self, cache: &'chain SiblingCache<'tree>) -> Self { + Self { + siblings: Some(cache), + ..self + } + } + + /// [`Ancestors::known`], but first checks that `chain` really is + /// `node`'s ancestry. + /// + /// The parity test below proves the walker's truncate/push rule on a + /// *replica* walker, so it cannot see `metrics_inner` itself + /// desynchronising — a `chain.push` moved ahead of the per-node + /// computes, say, or a `continue` inserted above the truncate. + /// [`Ancestors::parent`] trusts `chain.last()` unvalidated, so such a + /// drift would feed every predicate a wrong ancestor silently rather + /// than fail. Walkers that maintain a chain should construct through + /// here; [`Ancestors::known`] stays unchecked for the callers that + /// deliberately pair a chain with a foreign node. + /// + /// # Two checks, because the exact one is not affordable by default + /// + /// The invariant is "`chain.last()` **is** `node.parent()`", and + /// asking that outright costs [`Node::parent`]'s `O(depth)` — the + /// very lookup #1084 exists to remove. Per node, on every walk that + /// constructs a checked chain, it made every debug-build walk + /// `O(nodes × depth)` while the shipped walk is `O(nodes)`: a tax on + /// every `cargo test`, worst on the deep-nesting regression tests + /// that exist to pin the shipped walk's linearity (#1122). It now + /// runs only under `--cfg chain_audit`, which `make chain-audit` and + /// the CI lane of the same name set — and there as a plain + /// `assert_eq!`, so the audit has teeth in a release profile too. + /// + /// What stays on in every debug build is an `O(1)` *consequence* of + /// that invariant: a parent's byte span contains its child's, and no + /// node is its own parent. Strictly weaker — a grandparent contains + /// the node as well — but it is shaped to the two ways a walker + /// desynchronises. A `push` moved ahead of the per-node computes + /// leaves `chain.last() == node`; a dropped `truncate` leaves the + /// previous subtree's path, whose last entry is disjoint from the + /// node that follows it. Both trip here, on the first node that + /// shows them, at four integer comparisons. + #[must_use] + pub fn checked(chain: &'chain [Node<'tree>], node: &Node<'tree>) -> Self { + // Opt-in only: `Node::parent` restarts at the root, so this is + // the `O(nodes × depth)` walk described above. + #[cfg(chain_audit)] + assert_eq!( + chain.last().map(Node::id), + node.parent().map(|parent| parent.id()), + "ancestor chain desynchronised on a {} node", + node.kind() + ); + debug_assert!( + chain.last().is_none_or(|parent| { + parent.id() != node.id() + && parent.start_byte() <= node.start_byte() + && node.end_byte() <= parent.end_byte() + }), + "ancestor chain desynchronised on a {} node: chain.last() neither \ + contains it nor differs from it", + node.kind() + ); + Self::known(chain) + } + + /// How far the node this chain describes sits from the root, or + /// `None` when no chain is known — deriving it then would cost the + /// [`Node::parent`] climb the chain exists to remove. + #[inline] + pub fn depth(self) -> Option { + self.chain.map(<[Node<'tree>]>::len) + } + + /// `node`'s parent. + #[inline] + #[must_use] + pub fn parent(self, node: &Node<'tree>) -> Option> { + match self.chain { + Some(chain) => chain.last().copied(), + None => node.parent(), + } + } + + /// Whether `node`'s parent has kind id `kind`. + /// + /// The compound-leaf guard of `.claude/rules/grammar-dispatch.md` + /// section 5 asks this and nothing else: a delimiter or keyword + /// token is suppressed *only* directly under the construct that + /// owns it, never under an arbitrary ancestor. Eleven dispatch + /// arms across `getter`, `checker` and the metric walkers spelled + /// it out as `.parent(node).is_some_and(|p| p.kind_id() == X as + /// u16)`, which wraps onto four rustfmt lines inside a match guard + /// and buries the question under the plumbing; #1314 added six more + /// and folded all seventeen onto this. Three sites spelling their + /// binding `|parent|` were left alone and would each fit: + /// `impl_is_else_if_parent_clause!` (`src/checker.rs`), + /// `is_stabby_lambda_body` (`src/checker/ruby.rs`) and + /// `is_useful_comment` (`src/checker/rust.rs`, a let-chain). + /// + /// A `false` when `node` has no parent is the answer every one of + /// those call sites wants: a root node's token is not inside the + /// construct, so it is not suppressed. + #[inline] + #[must_use] + pub fn parent_has_kind(self, node: &Node<'tree>, kind: u16) -> bool { + self.parent(node).is_some_and(|p| p.kind_id() == kind) + } + + /// The sibling immediately before `node`, or `None` when `node` is + /// its parent's first child or has no parent. + /// + /// `tree_sitter`'s own `prev_sibling` resolves the parent first + /// (`ts_node__prev_sibling` opens with `ts_node_parent`), so it + /// carries the same `O(depth)` cost [`Node::parent`] does. With a + /// known chain the parent is free and what remains is a cursor walk + /// over the siblings — or, for a wide parent the walk recorded in its + /// [`SiblingCache`], a binary search, since a walk over every child + /// of a Kotlin string's `$name` run was quadratic in it (#1584). + #[inline] + #[must_use] + pub fn previous_sibling(self, node: &Node<'tree>) -> Option> { + let Some(chain) = self.chain else { + return node.previous_sibling(); + }; + let parent = chain.last()?; + if let Lookup::Wide(children, index) = self.lookup(node) { + return match index { + Some(index) => children[..index].last().copied(), + None => node.previous_sibling(), + }; + } + let mut previous = None; + for child in parent.children() { + if child.id() == node.id() { + return previous; + } + previous = Some(child); + } + // `node` is not among `parent.children()`, so this chain does + // not describe `node`. Answering `None` would claim "no + // previous sibling", which is a different (and wrong) answer; + // fall back to the authoritative lookup instead. + node.previous_sibling() + } + + /// `node`'s ancestors, nearest first, each paired with *its* own + /// ancestry so a predicate applied to an ancestor stays as cheap as + /// one applied to `node`. + #[inline] + #[must_use] + pub fn iter(self, node: &Node<'tree>) -> AncestorIter<'tree, 'chain> { + match self.chain { + Some(chain) => AncestorIter::Chain(chain, self.siblings), + None => AncestorIter::Climb(node.parent()), + } + } + + /// The siblings after `node`, nearest first: what + /// [`Node::children_after`] yields on `node`'s parent, and nothing + /// for the root. `self` is `node`'s chain. + /// + /// The seek [`Node::children_after`] runs is `O(index)` over a flat + /// child array, so a walk that asks of every child of a wide + /// `ERROR` is quadratic in it (#1584). A walk that lends a + /// [`SiblingCache`] has already collected a wide parent's children, + /// and the lookup is a binary search among them. + pub fn siblings_after( + self, + node: &Node<'tree>, + ) -> impl Iterator> + use<'tree, 'chain> { + let (wide, narrow) = match self.lookup(node) { + Lookup::Wide(children, index) => (index.map(|index| &children[index + 1..]), None), + Lookup::Narrow(parent) => (None, parent.map(|parent| parent.children_after(node))), + }; + let wide = wide.into_iter().flatten().copied(); + wide.chain(narrow.into_iter().flatten()) + } + + /// The siblings before `node`, nearest first: what + /// [`Node::children_before`] yields on `node`'s parent, and nothing + /// for the root. `self` is `node`'s chain; see + /// [`siblings_after`](Self::siblings_after). + pub fn siblings_before( + self, + node: &Node<'tree>, + ) -> impl Iterator> + use<'tree, 'chain> { + let (wide, narrow) = match self.lookup(node) { + Lookup::Wide(children, index) => (index.map(|index| &children[..index]), None), + Lookup::Narrow(parent) => (None, parent.map(|parent| parent.children_before(node))), + }; + let wide = wide.into_iter().flatten().rev().copied(); + wide.chain(narrow.into_iter().flatten()) + } + + // Where `node`'s siblings are read from: its parent's children as the + // walk recorded them and `node`'s index among them, or the parent to + // seek in when the walk recorded none. + fn lookup(self, node: &Node<'tree>) -> Lookup<'tree, 'chain> { + let parent = self.parent(node); + let recorded = match (self.siblings, self.chain, parent) { + (Some(cache), Some(chain), Some(parent)) => { + let recorded = cache.children(&parent, chain.len() - 1); + // A walk that stopped recording would answer the same, + // only quadratically, which no metric value shows. + debug_assert!( + recorded.is_some() || parent.child_count() <= WIDE_PARENT, + "a walk lending a SiblingCache must record every wide node on its \ + chain (#1584), but not this {}", + parent.kind() + ); + recorded + } + _ => None, + }; + match recorded { + Some(children) => Lookup::Wide(children, position_by_start(children, node)), + None => Lookup::Narrow(parent), + } + } +} + +enum Lookup<'tree, 'chain> { + Wide(&'chain [Node<'tree>], Option), + Narrow(Option>), +} + +/// Children past which a walk records a node's children for sibling +/// lookups. A seek over this many is a short scan however the grammar +/// stores them, so recording a narrower node would copy it for nothing. +/// +/// `pub` and `#[doc(hidden)]` only so the wide-parent fixtures in this +/// crate's and `big-code-analysis`'s tests size themselves from it: a +/// hand-copied width would silently go narrow if this were raised. +#[doc(hidden)] +pub const WIDE_PARENT: usize = 64; + +// `node`'s index in `children`, a parent's children in order. Their start +// bytes never decrease, so a binary search lands on the first child +// starting where `node` does; only zero-width children, which recovery +// inserts, can share that byte with it. +fn position_by_start(children: &[Node], node: &Node) -> Option { + let first = children.partition_point(|child| child.start_byte() < node.start_byte()); + children[first..] + .iter() + .take_while(|child| child.start_byte() == node.start_byte()) + .position(|child| child.id() == node.id()) + .map(|offset| first + offset) +} + +/// The children of each wide node on a walk's chain, recorded as the +/// walk pushes them, so that [`Ancestors::siblings_after`] and +/// [`Ancestors::siblings_before`] find a sibling in a flat `ERROR` +/// without a scan from its start (#1584). +/// +/// One slot per depth, keyed by the node's id. A walk descends into a +/// node only after pushing its children, so every wide node on the +/// chain has its slot; a slot left behind by a node the walk has moved +/// past holds another id and is not read. Recording copies what the +/// walk already holds, once per wide node. +#[derive(Debug, Default)] +pub struct SiblingCache<'tree>(Vec<(usize, Vec>)>); + +impl<'tree> SiblingCache<'tree> { + /// Records `children`, in order, as the children of `node`, which + /// sits at `depth` in the walk's chain. A node of no more than a + /// short scan's children is not recorded. + pub fn record( + &mut self, + node: &Node<'tree>, + depth: usize, + children: impl IntoIterator>, + ) { + if node.child_count() <= WIDE_PARENT { + return; + } + if self.0.len() <= depth { + self.0.resize_with(depth + 1, Default::default); + } + // An unused slot's id is 0, which no node has: tree-sitter's id + // is the address of the node's subtree. + let (id, recorded) = &mut self.0[depth]; + *id = node.id(); + recorded.clear(); + recorded.extend(children); + } + + // `parent`'s children, when the walk recorded them at `depth`. + fn children(&self, parent: &Node<'tree>, depth: usize) -> Option<&[Node<'tree>]> { + self.0 + .get(depth) + .filter(|(id, _)| *id == parent.id()) + .map(|(_, children)| children.as_slice()) + } +} + +/// Ancestor iterator returned by [`Ancestors::iter`], nearest first. +pub enum AncestorIter<'tree, 'chain> { + /// The not-yet-yielded prefix of a known chain, and the walk's + /// [`SiblingCache`] if it lent one. The prefix's last element is + /// the next ancestor, and the prefix before it is that ancestor's + /// own chain — so splitting from the back hands out both at once. + Chain(&'chain [Node<'tree>], Option<&'chain SiblingCache<'tree>>), + /// The next ancestor to yield, reached by climbing. + Climb(Option>), +} + +impl<'tree, 'chain> Iterator for AncestorIter<'tree, 'chain> { + type Item = (Node<'tree>, Ancestors<'tree, 'chain>); + + fn next(&mut self) -> Option { + match self { + Self::Chain(remaining, siblings) => { + let (&nearest, above) = remaining.split_last()?; + *remaining = above; + let ancestors = Ancestors { + chain: Some(above), + siblings: *siblings, + }; + Some((nearest, ancestors)) + } + Self::Climb(next) => { + let nearest = (*next)?; + *next = nearest.parent(); + Some((nearest, Ancestors::unknown())) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::node::Tree; + + // The ids of `nodes`, so a failing comparison prints which differ. + #[cfg(feature = "perl")] + fn ids<'a>(nodes: impl Iterator>) -> Vec { + nodes.map(|node| node.id()).collect() + } + + // `parent`'s children through `child(i)`, the raw walk that neither + // the seek nor the recorded array is built on. + #[cfg(feature = "perl")] + fn raw_children(parent: &Node) -> Vec { + let raw = parent.as_tree_sitter(); + (0..raw.child_count() as u32) + .filter_map(|i| raw.child(i)) + .map(|child| child.id()) + .collect() + } + + // `my @r = (` and a body never closed: recovery makes the root an + // `ERROR` whose children are the body's tokens, one flat array of + // well over `WIDE_PARENT` of them. + #[cfg(feature = "perl")] + fn flat_error() -> Tree { + let source = format!("my @r = ({}", "{\nlast,\n$a && $b,\n".repeat(20)); + Tree::new::(source.as_bytes()) + } + + /// A recorded wide parent answers every child's siblings, and its + /// previous sibling, from the recorded array, and answers exactly + /// what the raw walk does on either side of it — the `ERROR` whose + /// flat array made the seek `O(index)` (#1584). A node that is no + /// child of it gets nothing. + #[cfg(feature = "perl")] + #[test] + fn siblings_of_a_recorded_wide_parent_match_the_child_walk() { + let tree = flat_error(); + let root = tree.get_root(); + assert!(root.is_error(), "the fixture's tail must fail to parse"); + assert!(root.child_count() > WIDE_PARENT, "the ERROR must be wide"); + + let mut cache = SiblingCache::default(); + cache.record(&root, 0, root.children()); + let chain = [root]; + let ancestors = Ancestors::known(&chain).with_siblings(&cache); + let all = raw_children(&root); + + let (mut observed, mut expected) = (Vec::new(), Vec::new()); + for (index, child) in root.children().enumerate() { + let recorded_index = match ancestors.lookup(&child) { + Lookup::Wide(_, index) => index, + Lookup::Narrow(_) => None, + }; + observed.push(( + index, + recorded_index, + ids(ancestors.siblings_after(&child)), + ids(ancestors.siblings_before(&child)), + ancestors.previous_sibling(&child).map(|node| node.id()), + )); + expected.push(( + index, + Some(index), + all[index + 1..].to_vec(), + all[..index].iter().rev().copied().collect(), + index.checked_sub(1).map(|previous| all[previous]), + )); + } + assert_eq!(observed, expected); + + // A grandchild starting where its parent does, so the search + // reaches it and only the id comparison turns it away. + let stray = root + .children() + .find_map(|child| { + child + .child(0) + .filter(|g| g.start_byte() == child.start_byte()) + }) + .expect("the fixture nests a node at its parent's start"); + assert!(matches!(ancestors.lookup(&stray), Lookup::Wide(_, None))); + assert_eq!(ancestors.siblings_after(&stray).count(), 0); + assert_eq!(ancestors.siblings_before(&stray).count(), 0); + // `previous_sibling` falls back to the node's own lookup instead, + // which finds a stray's real previous sibling: `$b`'s is the `&&` + // of its own chain, not anything in the root. + let operand = root + .children() + .find_map(|child| child.child(2)) + .expect("the fixture holds `$a && $b`"); + let previous = ancestors + .previous_sibling(&operand) + .expect("the fallback finds the `&&`"); + assert_eq!(previous.kind(), "&&"); + } + + /// A narrow parent is never recorded, and its siblings come from + /// the seek — with or without a cache lent. + #[cfg(feature = "perl")] + #[test] + fn a_narrow_parent_is_sought_not_recorded() { + let tree = flat_error(); + let root = tree.get_root(); + let chain = [root]; + let narrow = root + .children() + .find(|child| (2..=WIDE_PARENT).contains(&child.child_count())) + .expect("the fixture holds a narrow node with several children"); + let mut cache = SiblingCache::default(); + cache.record(&narrow, 1, narrow.children()); + assert!(cache.0.is_empty(), "a narrow node must not be recorded"); + + let narrow_chain = [root, narrow]; + let all = raw_children(&narrow); + for ancestors in [ + Ancestors::known(&narrow_chain).with_siblings(&cache), + Ancestors::known(&narrow_chain), + ] { + let first = narrow.child(0).expect("narrow node has children"); + assert!(matches!(ancestors.lookup(&first), Lookup::Narrow(Some(_)))); + assert_eq!(ids(ancestors.siblings_after(&first)), all[1..]); + } + // Without a cache a wide parent is sought too. + let first = root.child(0).expect("root has children"); + assert!(matches!( + Ancestors::known(&chain).lookup(&first), + Lookup::Narrow(Some(_)) + )); + } + + /// A walk that lends a cache but did not record a wide node on its + /// chain would answer correctly and quadratically; the assertion is + /// what makes that visible to every test that walks one. + #[cfg(all(feature = "perl", debug_assertions))] + #[test] + #[should_panic(expected = "must record every wide node on its chain")] + fn an_unrecorded_wide_parent_trips_the_walk_assertion() { + let tree = flat_error(); + let root = tree.get_root(); + let cache = SiblingCache::default(); + let chain = [root]; + let first = root.child(0).expect("root has children"); + let _ = Ancestors::known(&chain) + .with_siblings(&cache) + .lookup(&first); + } + + /// The binary search finds every child at its index, including a + /// zero-width one: recovery inserts it at the byte its successor + /// starts at, so only the id tells the two apart. The Kotlin + /// fixture's `vararg: Int` holds one. + #[cfg(feature = "kotlin")] + #[test] + fn position_by_start_finds_a_zero_width_child() { + let tree = Tree::new::(b"fun f(vararg: Int, b: Int) {}"); + let mut zero_width = 0; + for node in tree.get_root().preorder() { + let children: Vec<_> = node.children().collect(); + for (index, child) in children.iter().enumerate() { + zero_width += usize::from(child.start_byte() == child.end_byte()); + assert_eq!(position_by_start(&children, child), Some(index)); + } + } + assert!( + zero_width > 0, + "fixture must hold an inserted zero-width child" + ); + } +} diff --git a/big-code-analysis-ast/src/parser.rs b/big-code-analysis-ast/src/parser.rs index d5e30b990..7a936653b 100644 --- a/big-code-analysis-ast/src/parser.rs +++ b/big-code-analysis-ast/src/parser.rs @@ -133,7 +133,7 @@ impl ParserTrait for P } fn filters(&self, requested: &[String]) -> Filter<'_> { - // Borrowed by the `"function"` and `"string"` arms below, which + // Borrowed by the `"call"`, `"function"` and `"string"` arms below, which // is why `Filter` carries a lifetime. let code = self.code(); let mut res: Vec>> = Vec::new(); @@ -141,13 +141,15 @@ impl ParserTrait for P let f = f.as_str(); match f { "all" => res.push(Box::new(|_: &Node, _| -> bool { true })), - // `is_call` / `is_comment` / `is_error` take `&Node` and - // nothing else, so no language *can* make them - // text-dependent. The #1162 gap is confined by - // construction to the `Checker` predicates that accept - // `code`, and `"function"` and `"string"` are the two - // filters that reach one. - "call" => res.push(Box::new(|node: &Node, _| T::is_call(node))), + // `is_call_with_code`, not `is_call`: Perl parses a + // call's name and an auto-quoted hash key as bareword + // calls, and only the chain tells those from a call of + // their own (#1580). `is_comment` / `is_error` take + // `&Node` and nothing else, so no language *can* make + // them text-dependent. + "call" => res.push(Box::new(move |node: &Node, ancestors| { + T::is_call_with_code(node, code, ancestors) + })), "comment" => res.push(Box::new(|node: &Node, _| T::is_comment(node))), "error" => res.push(Box::new(|node: &Node, _| T::is_error(node))), // `is_string_with_code`, not `is_string`: a Tcl-family diff --git a/big-code-analysis-bench/src/shapes.rs b/big-code-analysis-bench/src/shapes.rs index 4279b7566..ece4a13d0 100644 --- a/big-code-analysis-bench/src/shapes.rs +++ b/big-code-analysis-bench/src/shapes.rs @@ -34,6 +34,23 @@ use big_code_analysis::{Ast, CodeMetrics, LANG, Metric, MetricsError, MetricsOpt /// `base + per_unit * n`. See the module docs for why. pub type Render = fn(usize) -> String; +/// C#: a qualified type with one generic argument per segment. +#[must_use] +pub fn csharp_qualified_generics(depth: usize) -> String { + format!("class C {{ {}A field; }}\n", "A.".repeat(depth)) +} + +/// Elixir: multiple timeout clauses accepted by the grammar but rejected +/// by the compiler. An analyzer must still process them in linear time. +#[must_use] +pub fn elixir_many_timeouts(width: usize) -> String { + format!( + "receive(do: ({}x -> :a), after: ({}1 -> :t))\n", + "x -> :a; ".repeat(width), + "1 -> :t; ".repeat(width) + ) +} + /// Rust: `fn f() -> i32 { (((…1…))) }`. /// /// The paren chain is the shape that first surfaced both the `tokens` @@ -503,6 +520,29 @@ pub fn perl_dash_keys(width: usize) -> String { ) } +/// Perl: `sub f { return g(foo => 1, -foo => 1, …); }` — one call whose +/// argument list holds `width` bareword keys, alternately plain and +/// signed. +/// +/// The width shape for the #1572 bareword-key workaround, which asks of +/// every bareword call, and of every `-` before one, whether a `=>` +/// follows it. Unlike [`perl_dash_keys`], whose `-foo` the grammar reads +/// as a file test, neither key here has a letter that is a file test, so +/// this reaches the `-` and bareword-call path instead. +#[must_use] +pub fn perl_bareword_keys(width: usize) -> String { + let keys: Vec<&str> = (0..width) + .map(|i| { + if i % 2 == 0 { + "name => 1" + } else { + "-name => 1" + } + }) + .collect(); + format!("sub f {{ return g({}); }}\n", keys.join(", ")) +} + /// Perl: `my @r = (/a/, /a/, …);` — one list of `width` bare matches. /// /// The width shape for the Perl ABC pattern-role check, which asks of @@ -514,6 +554,36 @@ pub fn perl_pattern_list(width: usize) -> String { format!("my @r = ({});\n", vec!["/a/"; width].join(", ")) } +/// Kotlin: `val s = "$a $a …"` — one string holding `width` short +/// `$name` interpolations. +/// +/// The width shape for Kotlin Halstead's short-interpolation test, which +/// reads the token before every name. It found that token by scanning +/// the string's children from its start, quadratic in the name count on +/// valid input: 11.5 s for 8 000 names in a release build (#1584). +#[must_use] +pub fn kotlin_short_interpolations(width: usize) -> String { + format!("val s = \"{}\"\n", "$a ".repeat(width)) +} + +/// Elixir: `f(x, x, …, k: if(a, do: 1), k: if(a, do: 1), …)` — one call +/// with `width` positional arguments before `width` keyword pairs. +/// +/// The width shape for the keyword-section lookup #1574 added, which +/// asks of every pair whether its keyword list is the call's trailing +/// argument, and of every keyword-form clause which Call owns it. Its +/// first form re-read the argument list from its front per pair, +/// quadratic in a call with many of both: 5.4 s for 16 000 arguments in +/// a release build. The `if` in each value is the cognitive reading. +#[must_use] +pub fn elixir_mixed_arguments(width: usize) -> String { + format!( + "f({}{})\n", + "x, ".repeat(width), + vec!["k: if(a, do: 1)"; width].join(", ") + ) +} + /// Kotlin: `inline fun f(noinline p0: () -> Unit, …) {}` — one function /// with `width` modified parameters. /// @@ -534,6 +604,157 @@ pub fn kotlin_noinline_params(width: usize) -> String { source } +/// Perl: `my $r = $v || $v || … || $v;`. +/// +/// `||` is left-associative, so the chain is a left-nested spine one +/// `binary_expression` deeper per operand. Perl ABC asks every chain +/// node whether a misparsed comparison (#1273) absorbed its leading +/// operand, by climbing the spine of leading slots to its root. Every +/// node but the bottom one leads with a chain, which scores itself, so +/// only the bottom one climbs; climbing from each was `O(depth)` per +/// operator: ~15 s for 8 000 operands in a release build. +#[must_use] +pub fn perl_or_chain(depth: usize) -> String { + format!("my $r = {}$v;\n", "$v || ".repeat(depth)) +} + +/// Perl: `my $r = $x > $v && $v && … && $v;`. +/// +/// [`perl_or_chain`] behind the #1273 misparse it exists to detect: +/// tree-sitter-perl lets `>` swallow the chain to its right, so the +/// whole spine is the comparison's right operand and the bottom node's +/// climb ends on the `>`, answering "absorbed". The two shapes share +/// the climb and differ in where it stops. +#[must_use] +pub fn perl_misparsed_comparison(depth: usize) -> String { + format!("my $r = $x > {}$v;\n", "$v && ".repeat(depth)) +} + +/// Perl: `sub f { $a && $b; $a && $b; … }` — one block of `width` +/// two-operand chains. +/// +/// The width shape for the same misparse check. Each chain's leading +/// operand is a bare variable, so each climbs, and the climb stops one +/// step up at the block to read the operator before the chain. +/// Scanning the block's statements from its start for it was +/// `O(width)` per chain: 16.3 s for 10 000 statements in a release +/// build. +#[must_use] +pub fn perl_and_statements(width: usize) -> String { + format!("sub f {{ {}}}\n", "$a && $b; ".repeat(width)) +} + +/// Perl: `my @r = (` followed by `width` copies of +/// `last if $c =~ /x/;⏎{⏎$a && $b,⏎`, never closed. +/// +/// The width shape for a file whose tail fails to parse, as +/// `perl5db.pl`'s does: tree-sitter-perl gives up on the unclosed list, +/// the root becomes an `ERROR`, and the tail's tokens are its direct +/// children, one flat array. Perl ABC asks of each `=~` whether it binds +/// a rewrite, of each bareword (`last`) whether a `=>` follows it, and +/// of each `&&` chain which operator precedes it. Each question reads a +/// neighbour in the `ERROR`, and a seek by byte over a flat array scans +/// it from its start, so each was `O(width)` until the walk recorded the +/// `ERROR`'s children (#1584). +#[must_use] +pub fn perl_error_tail(width: usize) -> String { + format!( + "my @r = ({}", + "last if $c =~ /x/;\n{\n$a && $b,\n".repeat(width) + ) +} + +/// Perl: `my @r = (` followed by `width` copies of +/// `{⏎- and,⏎not,⏎-a => 1,⏎`, never closed. +/// +/// [`perl_error_tail`]'s flat `ERROR`, holding the key shapes Halstead +/// asks about: whether a `=>` follows each `-a`, and whether a `-` signs +/// each `and` / `not` key. +#[must_use] +pub fn perl_error_tail_keys(width: usize) -> String { + format!("my @r = ({}", "{\n- and,\nnot,\n-a => 1,\n".repeat(width)) +} + +/// Perl: `my @r = (` followed by `width` copies of +/// `{⏎last,⏎name => 1,⏎`, never closed. +/// +/// [`perl_error_tail`]'s flat `ERROR`, holding a bareword call and a +/// bareword key the grammar also reads as a call: the `call` filter +/// asks of each whether a `=>` follows it. +#[must_use] +pub fn perl_error_tail_calls(width: usize) -> String { + format!("my @r = ({}", "{\nlast,\nname => 1,\n".repeat(width)) +} + +/// Perl: `my @r = (` followed by `width` copies of +/// `{⏎die => 1,⏎exit => 1,⏎exit,⏎`, never closed. +/// +/// [`perl_error_tail`]'s flat `ERROR`, holding the `die` / `exit` keys +/// nexits asks about: whether a `=>` follows each, which makes it a key +/// rather than an exit. The bare `exit` is the one exit per copy. +#[must_use] +pub fn perl_error_tail_exit_keys(width: usize) -> String { + format!( + "my @r = ({}", + "{\ndie => 1,\nexit => 1,\nexit,\n".repeat(width) + ) +} + +/// Bash: `if [ "$a" = 0 -o "$a" = 0 -a … ]; then :; fi`, the +/// connectives alternating. +/// +/// tree-sitter-bash lets `=` swallow the connectives around it, so the +/// test parses as one left-nested spine of comparisons, one level per +/// term (#1536). Bash cognitive ends each connective's boolean +/// sequence where the outermost comparison of its run does; finding +/// that end by climbing the run from every `binary_expression` was +/// `O(depth)` per node: 1.9 s for 4 000 terms in a release build. The +/// connectives alternate so the reading — one sequence per switch, the +/// textual-order scoring the mis-nested tree forces — grows with the +/// depth. +#[must_use] +pub fn bash_test_connective_run(depth: usize) -> String { + let mut source = String::from("if [ "); + for i in 0..depth { + source.push_str(if i % 2 == 0 { + "\"$a\" = 0 -o " + } else { + "\"$a\" = 0 -a " + }); + } + source.push_str("\"$a\" = 0 ]; then :; fi\n"); + source +} + +/// Bash: `(( a = a = … = 0 ))`. +/// +/// A valid chained arithmetic assignment, one `binary_expression` +/// deeper per `=`. The same `=` token is a string comparison inside +/// `[ … ]` / `[[ … ]]`, and Bash ABC once told the two apart by +/// climbing from every `=` through the expression nodes above it to +/// see whether a test held them — `O(depth)` per `=` here: 3.4 s for +/// 8 000 in a release build. Each test now counts its own comparisons +/// from the top, so an arithmetic `=` asks nothing. +#[must_use] +pub fn bash_arith_assign_chain(depth: usize) -> String { + format!("(( {}0 ))\n", "a = ".repeat(depth)) +} + +/// Bash: `if [[ $a = 0 && $a = 0 && … ]]; then :; fi`. +/// +/// The `[[ … ]]` form of the [`bash_test_connective_run`] misparse: +/// `=` swallows the `&&` around it, so the test is one deep spine. It +/// is the shape the comparison-counting change in Bash ABC has to stay +/// linear on, since every `=` in it is a comparison: 1.2 s for 16 000 +/// terms in a release build while each `=` climbed to its test. +#[must_use] +pub fn bash_eq_and_test(depth: usize) -> String { + format!( + "if [[ {}$a = 0 ]]; then :; fi\n", + "$a = 0 && ".repeat(depth) + ) +} + /// C++: `int f(int a){ return a + a + … + a; }`. /// /// The shape control for [`cpp_and_chain`]: the same left-nested @@ -547,9 +768,10 @@ pub fn cpp_plus_chain(depth: usize) -> String { /// The walk a probe times. /// -/// Two seams reach the AST walker — `Ast::metrics` and `Ast::ops` — and -/// they share no options type, so the selection a metric probe needs -/// lives on the variant that uses it rather than on every probe (#1110). +/// Three seams reach an AST walk — `Ast::metrics`, `Ast::ops` and the +/// `Ast::count` walk `bca find` shares — and they share no options type, +/// so the selection a metric probe needs lives on the variant that uses +/// it rather than on every probe (#1110). #[derive(Clone, Copy)] pub enum Workload { /// `Ast::metrics` under `selection`. @@ -575,12 +797,18 @@ pub enum Workload { /// Headline value the walk produces on the probe's shape. reading: fn(&Ops) -> u64, }, + /// `Ast::count` under one `--type` filter, the walk `bca count` and + /// `bca find` share. Its reading is the matching node count. + Count { + /// The filter, as `bca count --type` spells it. + filter: &'static str, + }, } impl Workload { /// The options this workload walks under, resolved once so option - /// resolution stays outside the timed region. `Ast::ops` takes none, - /// so the default stands in and is ignored. + /// resolution stays outside the timed region. `Ast::ops` and + /// `Ast::count` take none, so the default stands in and is ignored. #[must_use] pub fn options(self) -> MetricsOptions { match self { @@ -591,7 +819,7 @@ impl Workload { } => MetricsOptions::default() .with_only(selection) .with_exclude_tests(exclude_tests), - Self::Ops { .. } => MetricsOptions::default(), + Self::Ops { .. } | Self::Count { .. } => MetricsOptions::default(), } } @@ -609,6 +837,10 @@ impl Workload { Ok(match self { Self::Metrics { reading, .. } => reading(&black_box(ast.metrics(options)?).metrics), Self::Ops { reading } => reading(&black_box(ast.ops()?)), + Self::Count { filter } => { + let (matching, _) = black_box(ast.count(&[filter.to_owned()])); + u64::try_from(matching).unwrap_or(u64::MAX) + } }) } } @@ -766,6 +998,36 @@ const LINEAR_BOUND: f64 = 1.5; /// /// See `docs/development/benchmarking.md` for the per-probe table. pub const PROBES: &[Probe] = &[ + Probe { + name: "halstead/csharp-qualified-generics", + lang: LANG::Csharp, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Halstead], + reading: |m| m.halstead.total_operators(), + }, + render: csharp_qualified_generics, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "Each generic qualifier must be classified without climbing \ + the remaining qualified-name ancestry.", + }, + Probe { + name: "abc/elixir-many-timeouts", + lang: LANG::Elixir, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc, Metric::Cyclomatic], + reading: |m| m.abc.conditions_sum(), + }, + render: elixir_many_timeouts, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "Malformed receive timeout clauses must not each rescan every \ + message clause. The reading comes from the message clauses.", + }, Probe { name: "tokens/nested-paren", lang: LANG::Rust, @@ -1021,6 +1283,29 @@ pub const PROBES: &[Probe] = &[ ancestor, and each step now indexes the walker's \ ancestor chain rather than calling `Node::parent`.", }, + Probe { + name: "cognitive/elixir-mixed-arguments", + lang: LANG::Elixir, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Cognitive], + reading: |m| m.cognitive.cognitive_sum(), + }, + render: elixir_mixed_arguments, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1574's keyword-section lookup asks of every keyword \ + pair whether its list is the call's trailing argument \ + (cognitive), and of every keyword-form clause which \ + Call owns it (cyclomatic, ABC). Its first form found \ + the trailing argument by reading the argument list \ + from its front, once per pair: 0.10 s, 0.36 s, 1.4 s \ + and 5.4 s for 2 000 to 16 000 arguments in a release \ + build, exponent 1.94. The lookup now reads the list's \ + siblings off the walk's chain. The reading is one \ + `if` per pair.", + }, Probe { name: "nom/nested-fn", lang: LANG::Rust, @@ -1355,6 +1640,28 @@ pub const PROBES: &[Probe] = &[ ABC asks through the same helper. The reading is the \ operand count, which grows with the width.", }, + Probe { + name: "abc/perl-bareword-keys", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc, Metric::Halstead], + reading: |m| m.halstead.total_operands(), + }, + render: perl_bareword_keys, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1572's bareword-key workaround asks of every bareword \ + call (ABC) and of every `-` before one (Halstead) \ + whether a `=>` follows it, the question \ + `halstead/perl-dash-keys` pins for file-test keys. \ + Finding that `=>` by scanning the list from its start \ + would be quadratic in the key count, as #1545's first \ + form was. The reading is the operand count, which \ + grows with the width; the branch count would not, \ + since the fix leaves only `g` scoring one.", + }, Probe { name: "abc/perl-pattern-list", lang: LANG::Perl, @@ -1376,6 +1683,169 @@ pub const PROBES: &[Probe] = &[ pattern. The reading is one condition per bare \ match, so it grows with the width.", }, + Probe { + name: "abc/perl-or-chain", + lang: LANG::Perl, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc], + reading: |m| m.abc.conditions_sum(), + }, + render: perl_or_chain, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1273's misparse check climbs a chain's spine of \ + leading slots to ask whether a comparison absorbed \ + its first operand. A node whose leading slot scores \ + itself answers at once, so only the bottom of a \ + left-nested `||` spine climbs. Removing that early \ + answer, as before the fix, takes this probe from 1.00 \ + to 2.00 (8.8 ms to 2.33 s at depth 4 000) and \ + `abc/perl-misparsed-comparison` to 2.01, while every \ + other probe holds. The reading is one condition per \ + operand.", + }, + Probe { + name: "abc/perl-misparsed-comparison", + lang: LANG::Perl, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc], + reading: |m| m.abc.conditions_sum(), + }, + render: perl_misparsed_comparison, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "The same climb as `abc/perl-or-chain`, on a chain \ + the #1273 misparse really absorbed, so the bottom \ + node's climb ends on the `>` and answers yes. The two \ + probes fail together when the climb runs from every \ + node. The reading is one condition per operand.", + }, + Probe { + name: "abc/perl-and-statements", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc], + reading: |m| m.abc.conditions_sum(), + }, + render: perl_and_statements, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "The misparse check's climb from a two-operand \ + chain stops one step up, at the block, and asks for \ + the operator just before the chain. Scanning the \ + block from its start for it, as before the fix, takes \ + this probe from 1.08 to 2.01 (3.7 ms to 306 ms at \ + 2 000 statements). The kind gate that now skips a \ + block is not what this probe sees: with the gate alone \ + removed the lookup seeks the child by byte through \ + the block's balanced repeat and the probe holds at \ + 1.19. The reading is two conditions per statement.", + }, + Probe { + name: "abc/perl-error-tail", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc], + reading: |m| m.abc.conditions_sum(), + }, + render: perl_error_tail, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "In a file whose tail fails to parse, as in \ + `perl5db.pl`, every `=~`, bareword and `&&` chain of \ + the tail is a child of one flat `ERROR`. \ + `perl_binding_rewrites` reads each `=~`'s parent's \ + last operand, the bareword-key test the token after \ + each `last`, and the misparse check the token before \ + each chain. Reading those off the `ERROR` from its \ + start made this probe fit 2.09 (#1540, #1584). A \ + shape that does not parse (`RECOVERY_PROBES`). The \ + reading is four conditions per copy of the body.", + }, + Probe { + name: "halstead/perl-error-tail-keys", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Halstead], + reading: |m| m.halstead.total_operands(), + }, + render: perl_error_tail_keys, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "`abc/perl-error-tail`'s flat `ERROR`, holding the \ + `-and`, `not` and `-a` keys Halstead bills: the key \ + test asks for the token after each, and the sign \ + test for the token beside it. Read off the `ERROR` \ + from its start, as before #1584, both were \ + quadratic in the tail. The reading is three operands \ + per copy of the body.", + }, + Probe { + name: "nexits/perl-error-tail-keys", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Nexits], + reading: |m| m.nexits.nexits_sum(), + }, + render: perl_error_tail_exit_keys, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "`abc/perl-error-tail`'s flat `ERROR`, holding the \ + `die` and `exit` keys nexits must not score: the key \ + test asks for the token after each (#1585), which \ + the walk's `SiblingCache` answers; read off the \ + `ERROR` from its start it would be quadratic in the \ + tail. The reading is one `exit` per copy of the body.", + }, + Probe { + name: "count/perl-error-tail-calls", + lang: LANG::Perl, + axis: Axis::Width, + workload: Workload::Count { filter: "call" }, + render: perl_error_tail_calls, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "`abc/perl-error-tail`'s flat `ERROR` under the walk \ + `bca count` and `bca find` share. The `call` filter \ + asks the bareword-key test of every bareword (#1580), \ + which reads the token after it; that walk lends no \ + `SiblingCache` unless `act_on_node` records one, and \ + a seek from the `ERROR`'s start is quadratic in the \ + tail. The reading is one call, `last`, per copy of \ + the body.", + }, + Probe { + name: "halstead/kotlin-short-interpolations", + lang: LANG::Kotlin, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Halstead], + reading: |m| m.halstead.total_operands(), + }, + render: kotlin_short_interpolations, + sizes: LINEAR_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "Kotlin Halstead asks of every `$name` whether a `$` \ + token precedes it. `Ancestors::previous_sibling` \ + answered by scanning the string's children from its \ + start, quadratic in the name count on valid input, \ + until it read the children the walk recorded \ + (#1584). The reading is one operand per name plus \ + `s`.", + }, Probe { name: "halstead/kotlin-noinline-params", lang: LANG::Kotlin, @@ -1503,6 +1973,68 @@ pub const PROBES: &[Probe] = &[ inherits any climb added there. The reading is the \ chain's operand count.", }, + Probe { + name: "cognitive/bash-test-connective-run", + lang: LANG::Bash, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Cognitive], + reading: |m| m.cognitive.cognitive_sum(), + }, + render: bash_test_connective_run, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1536: each `binary_expression` of a mis-nested \ + `[ … -o … ]` run asks where its boolean sequence ends, \ + which is the end of the outermost comparison of the \ + run. Climbing the run from every node for it, as \ + before the fix, takes this probe from 0.98 to 2.06 \ + (6.5 ms to 1.99 s at depth 4 000); the run is now \ + tracked top-down. The reading is one sequence per \ + switch between `-o` and `-a`.", + }, + Probe { + name: "abc/bash-arith-assign-chain", + lang: LANG::Bash, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc, Metric::Halstead], + reading: |m| m.halstead.total_operators(), + }, + render: bash_arith_assign_chain, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1536: Bash ABC once climbed from every `=` to see \ + whether a test holds it, which on a valid chained \ + arithmetic assignment is the whole spine per `=`. \ + Restoring that climb takes this probe from 1.00 to \ + 1.97 (0.79 ms to 72.5 ms at depth 4 000) and \ + `abc/bash-eq-and-test` to 1.91. ABC counts no \ + assignment for an arithmetic `=`, so the reading is \ + the Halstead operator count, which grows with the \ + depth.", + }, + Probe { + name: "abc/bash-eq-and-test", + lang: LANG::Bash, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Abc], + reading: |m| m.abc.conditions_sum(), + }, + render: bash_eq_and_test, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1536: the `[[ … = … && … ]]` misparse, where every \ + `=` is a comparison ABC counts. Each test now counts \ + its own from the top; the per-`=` climb it replaced \ + takes this probe from 1.02 to 1.91 (2.9 ms to 75.2 ms \ + at depth 4 000). The reading is one condition per \ + `=` plus the `if`.", + }, ]; #[cfg(test)] @@ -1511,6 +2043,15 @@ mod tests { use super::{Axis, PROBES, Probe, Workload}; + /// Probes whose shape deliberately fails to parse; see + /// [`shapes_parse_without_error`]. + const RECOVERY_PROBES: &[&str] = &[ + "abc/perl-error-tail", + "halstead/perl-error-tail-keys", + "nexits/perl-error-tail-keys", + "count/perl-error-tail-calls", + ]; + /// Deepest `tree_sitter` node depth reachable from the root. /// /// Iterative on an explicit stack: the shapes here nest thousands @@ -1596,16 +2137,34 @@ mod tests { /// parser limit — a recursion cap, a token-count ceiling — is /// reached at scale and not on a toy input, so a small fixed size /// answers a different question than the gate asks. + /// + /// The probes in [`RECOVERY_PROBES`] are the inverse: their shape + /// *is* error recovery, and the cost they price is reading the + /// `ERROR` root's children. A grammar that learned to parse one + /// would leave it timing an ordinary narrow tree, so each must still + /// recover into an `ERROR` root. #[test] fn shapes_parse_without_error() { for probe in PROBES { let size = probe.sizes[0]; let ast = parse(probe, size); + let root = ast.as_tree_sitter().root_node(); + let (parses_as_declared, declared) = if RECOVERY_PROBES.contains(&probe.name) { + (root.is_error(), "into an ERROR root") + } else { + (!root.has_error(), "without an ERROR node") + }; + let name = probe.name; + let head: String = (probe.render)(size).chars().take(200).collect(); assert!( - !ast.as_tree_sitter().root_node().has_error(), - "{}: shape must parse without an ERROR node at size {size}:\n{}…", - probe.name, - (probe.render)(size).chars().take(200).collect::(), + parses_as_declared, + "{name}: shape must parse {declared} at size {size}:\n{head}…" + ); + } + for name in RECOVERY_PROBES { + assert!( + PROBES.iter().any(|probe| probe.name == *name), + "RECOVERY_PROBES names {name}, which is not in PROBES", ); } } diff --git a/big-code-analysis-book/src/commands/nodes.md b/big-code-analysis-book/src/commands/nodes.md index 0117e4fb3..bc25ad251 100644 --- a/big-code-analysis-book/src/commands/nodes.md +++ b/big-code-analysis-book/src/commands/nodes.md @@ -60,6 +60,10 @@ with a literal `call` node cannot be matched by name — use its numeric in C#, Java, Groovy, Kotlin, C++, JavaScript, TypeScript, PHP, Ruby and Rust; see [Where a constructor call lands](../metrics.md#abc-constructor-attribution). + Each call matches once. In Perl, where the grammar parses a call's + name as a call of its own, `foo(1)` matches as the whole call only, + and an auto-quoted hash key (`(name => 1)`, `$h{-name}`), which it + parses as a call too, does not match at all. - `comment` — a comment of any of the language's forms. - `string` — a string *literal*. Type-annotation keywords that share the literal's node name (TypeScript's and PHP's `: string`) are excluded. diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 19d7c6c57..050cb6f9a 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -158,11 +158,12 @@ application would over-count. | Kotlin | A subject-less `when` arm scores through the predicate slot; a subject-ful arm scores per entry | A subject-less arm's condition (`when { x > 5 -> … }`) is an ordinary boolean expression, compiled as an `if` predicate, so the comparison inside it is already counted by the operator arms and a blanket per-entry increment double-counted it. A subject-ful arm (`when (x) { in 1..2 -> … }`) lists a pattern rather than an independent boolean expression, so the implicit `subject == pattern` is a decision nothing in the source spells and the entry itself pays for it (#1421). | | Ruby | A subject-less `case` scores each `when` pattern as an `\|\|` operand; a subject-ful `case` scores per clause | A subject-less clause (`case; when x > 5`) lists ordinary boolean expressions, evaluated like an `if` predicate, and several patterns (`when a, b`) are an implicit `\|\|`. Each pattern therefore scores once unless its own operator already counted it, so `when x > 5` scores 1 like `if x > 5`, and `when a, b` scores 2 like `if a \|\| b`. Before this, a blanket per-clause increment counted a comparison twice. A subject-ful clause (`case x; when 1`) matches each pattern with an implicit `pattern === x` that nothing in the source spells, so the clause pays once (#1453). | | Go | A tagless `switch` scores each case expression as an `\|\|` operand; a tagged `switch` scores per case | A tagless switch (`switch { case x > 5: }`, with or without an init statement) is `switch true`: each case expression is an ordinary boolean scored by the same slot an `if` predicate uses, and a list (`case a, b:`) is an implicit `\|\|`. So `case x > 5:` scores 1 like `if x > 5`, and `case a, b:` scores 2 like `if a \|\| b`. Before this, a blanket per-case increment counted a comparison twice. A tagged case (`switch x { case 1: }`) compares with an implicit `x == 1` that nothing in the source spells, so the case pays once; type-switch and `select` cases are unchanged (#1523). | -| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive` (its `after` timeout included), of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `after:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). | -| C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these brackets as an operator (a generic list's as one `<>` pair since #1559), and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | +| Elixir | An `if` / `unless` predicate and each `cond` clause are condition slots; every other clause construct scores per clause; an `if` / `unless` `else` scores one | An `if` predicate scores one condition unless an operator inside it already counts it, as Ruby's does: `if x > 5` and `if b` each score 1, and `if a && b` scores 2 — its two operands. `cond` is an `if` / `else if` chain written as clauses, so each clause's condition is scored the same way, and `cond do x > 5 -> …; b -> …; true -> … end` scores 2, like Go's tagless `switch` twin. The unguarded `true ->` catch-all is `cond`'s default arm and scores nothing, as in cyclomatic. Before this, the keyword call added one condition on top of its predicate, so `if x > 5` scored 2 and `cond` scored once however many clauses it had (#1527). The same change scores every `&&` / `\|\|` operand and guard that no operator counts, so `a && &f/1` now scores 2, like `a && b`. The clauses of a `case`, of a `with`'s `else`, of a `receive`, of a `try`'s `rescue` / `catch` / `else`, and of a multi-clause `fn` after its head each score one, through the same gate cyclomatic uses: a bare `_ ->` catch-all outside a `fn` scores nothing, like Go's `default:` and Rust's `_ =>`. A typespec's function type (`@spec f((any -> any)) :: list`) parses as a clause too and scores nothing in either metric; cyclomatic counted it before. The keyword forms (`case(x, do: (1 -> :a; …))`, `cond(do: (…))`, `with`'s `else:`) hold their clauses in the same parenthesised block a typespec does, so they are recognised by the `do:` / `else:` / `rescue:` / `catch:` key that holds the block, and score exactly as the `do … end` forms. A clause whose pattern an operator already counts does not pay again, so `rescue e in RuntimeError ->` scores 1 — its `in`. So a two-clause `case` scores 2, like Ruby's subject-ful `case` and Rust's `match`; it scored 1 once per construct. Each `pattern <- expr` clause of a `with` scores one as well, because it either matches or falls through to the `else`, so `with {:ok, a} <- g(x) do a end` scores 1, like Rust's `if let Ok(a) = g(x)`. A `when` guard on that pattern is a guard slot like any other. A `for` generator's `<-` is not a clause and scores nothing. A `for … reduce:` block's accumulator clauses score like a multi-clause `fn`'s: the first is free, so a single `acc -> …` clause scores nothing (#1535). The `else` of an `if` / `unless`, in the block or the `else:` keyword form, is Fitzpatrick's Rule 5 `else` and scores one, as Ruby's, Java's, Python's and Go's do (#1531). A bracketed keyword list in last position (`if(c, [do: 1, else: 2])`) is the same argument as the bare one and scores the same, as does a quoted key (`"else": 2`); a list anywhere else, or one with a non-pair item, holds no sections. A `Kernel.&&` / `Kernel.\|\|` remote call scores its operands as `a && b` / `a \|\| b` does, the piped value of `a \|> Kernel.\|\|(b)` included (#1574), and so do `Kernel.and` / `Kernel.or` (#1582). A `receive` has one free arm, as a `case` does: its bare `_ ->` when the `do` section holds one, otherwise its `after` clause — the timeout, the path taken when no message clause matched in time — which then scores nothing; beside a bare `_ ->` the timeout is a path of its own and scores one (#1581). | +| C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TSX, JavaScript and Mozjs — TypeScript carries the same gate through the shared macro, but the `.ts` dialect of tree-sitter-typescript has no JSX production, so it has nothing to exclude there; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The declared name of an operator overload is excluded on every spelling too: C# overloads six comparison operators and gives each a distinct token, so `operator <=`, `operator >=`, `operator ==` and `operator !=` join `operator <` and `operator >`, which alone were excluded before #1420. C++ and Mozcpp do the same for their seven, `operator<=>` included, and also exclude the operator a comparison *fold* spells twice: `(0 == ... == a)` counts once, like `(a == ...)` (#1448). An `&&` / `\|\|` fold applies its operator once as well, and scores what one application of it scores: `(true && ... && a)` what `true && a` does, and `(... && a)`, whose unwritten side is one more operand, what `a && a` does (#1533). Their ISO alternative tokens `not_eq`, `not`, `and` and `or` count exactly as `!=`, `!`, `&&` and `\|\|` do (#1470). The exclusion is on the operator alone, not on the test that encloses it: since #1461 the `is` test itself is a condition wherever it is written, so `bool b = x is > 0;` and `return x is > 0;` each score 1 — level with the `x > 0` they are sugar for, and the same 1 a `when n is > 5` guard has scored since #1422 made the guard a condition slot. Counting the pattern's operator as well would make a relational arm worth twice the constant arm `5 => 1`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). The exclusion is ABC's alone: Halstead counts vocabulary rather than decisions, so `bca ops` still bills every one of these brackets as an operator (a type-enclosing pair as one `<>` since #1559 and #1567), and the two metrics are meant to disagree here (#1395) — see [the Halstead delimiter rule](#halstead). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | | C#, Java, Groovy, Kotlin, Ruby, Elixir, Perl | A relational operator scores by use; a value-bearing operand scores in a boolean slot | Five of these grammars spell at least one relational construct as its own production rather than as a binary expression with an operator token: C#'s `x is int` and `x is null`, Java's and Groovy's `x instanceof T`, Groovy's `a in l`, Kotlin's `a is T` and `a in 1..2`, and Ruby's one-line `a in Integer`. Having no token to count, each was reached only through the language's terminal-operand set, which the walker consults inside an `if` / `while` / ternary / `&&`-operand slot and nowhere else — so `var b = x == 1;` scored 1 while `var b = x is int;` scored 0. Fitzpatrick's Rule 5 counts a relational operator wherever it appears, so each now counts wherever it appears and has left the operand set; being in both would score it twice. Groovy's spaceship `<=>` counts on the same rule, as it already did in Ruby, PHP, C++ and Mozcpp, although its result is an integer rather than a boolean — what Rule 5 measures is the comparison, not its type. Elixir reached the same 0 by the other route: its membership and type tests (`a in [1, 2]`, `rescue e in RuntimeError`) *do* carry an operator token, and simply had no arm matching it. They count by use on the same rule, gated on the token's parent so that an operator merely *named* (`&in/2`) stays excluded alongside `<` and `>` — and, since #1531, alongside `==`, `===`, `!=`, `!==`, `<=` and `>=`, so `&==/2` and `Kernel.==(a, b)` score nothing. Perl's bare pattern match (`my $r = /^#/;`, or `m{^#}`) matches the implicit `$_` with no operator token at all, and scores by use on the same rule, level with `$x =~ /^#/` (#1467). Two patterns are excluded: the one bound by `=~` / `!~`, whose operator already counts, and the one `split` takes first (`split /,/, $s`), which is a delimiter rather than a match. A substitution or transliteration (`s///`, `tr///`, `y///`) edits its string and yields a count, so it is no test by use in either spelling: bare on `$_` or bound as `$x =~ s/a/b/`, it scores nothing outside a slot, and inside one the slot pays for it once (#1540). The converse still holds for a construct whose *value* fills the slot: a cast, a Go type assertion, a Rust `matches!` / `cfg!` macro and a Python walrus are all operands and count only where a slot reads them as a predicate (#1461). | | Perl | A comparison's right operand is not a condition, although the grammar parses it as one | tree-sitter-perl 1.1.2 lets a chaining comparison (`<`, `>`, `<=`, `>=`, `lt`, `gt`, `le`, `ge`, `==`, `!=`, `eq`, `ne`) take everything of lower precedence to its right as its operand, so `$a > $b ? 1 : 2` parses as `$a > ($b ? 1 : 2)` and `$a > $b && $c` as `$a > ($b && $c)`. The ternary's condition slot and the chain's first operand then held `$b`, which scored as a condition of its own: one above the JavaScript twin for each misparsed comparison. The walker recognises the shape and leaves that operand to the comparison, so both spellings score what the parenthesised `($a > $b) ? 1 : 2` and `($a > $b) && $c` do. An explicit `$a > ($b ? 1 : 2)` is unaffected. `<=>`, `cmp`, `=~` and `!~` do not chain and parse correctly. The workaround goes when the grammar is fixed upstream (#1273). | +| Perl | An auto-quoted hash key is a string, not a call | Perl quotes a bareword before `=>` and alone in a hash subscript, a leading `-` included, so `(name => 1)`, `new(name => $x)`, `(-name => 1)` and `$h{-name}` hold the strings `"name"` and `"-name"`. tree-sitter-perl 1.1.2 parses each word as a call, which scored a branch, and each `-name` as a negation, which billed a Halstead `-` operator and the operand `name`. Each now scores as its quoted twin `('name' => 1)` does, and Halstead bills the key as one operand spelled as written: `name`, `-name`. A package-qualified word (`Foo::Bar => 1`), which `=>` does not quote, a word with arguments (`name() => 1`) and a bareword value (`(a => foo)`) keep their call. `bca find --type call` reports the same calls ABC counts (#1580). Nor does `nexits` count a `die` or `exit` key as an exit (#1585). The workaround goes when the grammar is fixed upstream (#1571, #1572). | #### Where a constructor call lands {#abc-constructor-attribution} @@ -366,14 +367,25 @@ Sonar ecosystem. ### Per-language deviations - **Elixir** does not score recursion or jump statements. Elixir - control flow (`if` / `unless` / `cond` / `case` / `with`) is built + control flow (`if` / `unless` / `cond` / `case` / `with` / + `receive`) is built from macro-shaped `Call` nodes rather than dedicated grammar productions, and the language has no `break` / `continue` / `goto`; the implementation therefore scores only the nesting-bearing constructs it can identify and omits the recursion (B3) and unstructured-jump (B2) increments that the SonarSource specification adds for languages that expose those - shapes syntactically. + shapes syntactically. A `with`'s `else` adds nothing: its clauses + are the arms of the `case` the construct is in all but spelling, + already paid by the `with`'s own +nesting, as Ruby's `case`/`else` + is. An `if` / `unless` / `try` `else` still adds +1. The keyword + forms (`else:`, `rescue:`, `catch:`, bare or bracketed) score as + their `do … end` blocks do, and `Kernel.||(a, b)` is a boolean + sequence like `a || b` (#1574), as `Kernel.or(a, b)` is like + `a or b` (#1582). A `receive` is a `case` over the + mailbox and pays +nesting as one; its `after` timeout adds nothing, + and a `receive` holding only an `after` is a timed wait that scores + nothing (#1581). - For every language with a syntactic function-definition node, a nested function — a local function, or a method on a local / inner class — **resets the nesting counter to zero** at its @@ -517,6 +529,16 @@ one of them is a decision: `else _ -> 0` selects nothing, like a plain or diverges, so it scores one in both variants, as its `if let` twin does. Before #1542 it scored nothing. +Elixir's `receive` is a `case` over the mailbox, and modified scores +its clauses once, as it does a `case`'s. Like a `case` it has one free +arm: its bare `_ ->` when the `do` section holds one, otherwise its +`after` clause — the timeout, the path taken when no clause matched in +time. So `receive do {:a, y} -> y after 10 -> :t end` scores one +decision in standard, as the two-clause `case` does, and +`receive do {:a, y} -> y; _ -> 0 after 10 -> :t end` two: beside the +catch-all the timeout is a path of its own. A `receive` holding only an +`after` is a timed wait and scores nothing at all (#1581). + ### Counting Rust's `?` operator By default Rust's `?` operator (the `try_expression` grammar node) @@ -693,15 +715,20 @@ tokens involved the same way they spell real operators: `:` that opens a Python format spec (`f"{x:>10}"`) counts nothing either, because the spec after it is string content (#1486). Nor does the self-documenting `=` of `f"{x=}"`, which prints the - expression's text and assigns nothing (#1551). + expression's text and assigns nothing (#1551). C#'s format `:` and + alignment `,` (`$"{x,8:F2}"`) count nothing for the same reason + (#1573), but the alignment is an expression and keeps its own + vocabulary: `8` is an operand, and `{x,-w}` counts `-` and `w`. A + Python literal width (`f"{x:>10}"`) is format-spec text and counts + nothing. - **A delimiter around *syntax* is an operator, whatever it delimits.** The rule above is about a literal's own punctuation, and it stops there. Halstead counts the vocabulary a program is written in and has no notion of a decision, so a `<` is the same vocabulary entry whether it is a comparison, a JSX tag delimiter, a Lua `` - attribute, the declared name of a C# `operator <`, a Kotlin - `super`, or a Perl `` readline — and `bca ops` reports every - one of them. This is the deliberate counterpart to + attribute, the declared name of a C# `operator <`, or a Perl `` + readline — and `bca ops` reports every one of them. This is the + deliberate counterpart to [the ABC deviation](#abc-per-language-deviations) that excludes the same tokens from `conditions`: ABC asks whether a branch is taken, and these are not branches; Halstead asks what alphabet the source uses, @@ -710,45 +737,119 @@ tokens involved the same way they spell real operators: slice, dict and annotation `:`, and Rust's annotation, field, bound and label `:`, which is a separate operator from `::`. Both languages left these out until #1486, so their Halstead scores rose with it. - Python's `class` keyword followed in #1550: it is now billed like - `def`, as Ruby, Kotlin, C# and PHP bill their `class`. Java, Groovy, - the JavaScript family, C, C++ and Objective-C followed in #1552 with - their type-declaration keywords: `class`, `struct`, `union`, `enum`, - `interface`, `record`, `trait`, `@interface`, `@implementation` and - `@protocol`. Java's `C.class` is not billed: it stays one operand. - Kotlin's `interface` followed in #1556, and since #1558 Kotlin bills - every modifier, as C# does (Java and Groovy bill `abstract` and - `final` but no visibility keyword or `static`, #1565): the `enum` of - `enum class`, `data`, `sealed`, the visibility keywords, `open`, - `override`, `suspend`, `reified`, `out` and the rest each count once, - as the keyword and never as its `modifiers` wrapper. The same words - used as names bill no operator: where the grammar parses the name - (`val open = 1`) it is an operand, and where it cannot (`open = !open` - at the start of a statement, `val suspend = 1`, a parameter named - `vararg`) the word bills nothing. #1557 added C++'s `namespace`, `template`, `typename`, - `typedef` and `using` (C and Objective-C bill `typedef` too), so - `template ` and `template ` each bill their - keyword and score alike. It also added TypeScript's `type`, - `namespace`, `module`, `declare` and `global`, and made every - TypeScript type name an operand (`x: Foo`, `interface I`, - `Map`), as C#, Rust and the C family already did. Java followed - in #1560: `Foo f`, `implements I`, `List` and each segment of - `java.util.List` bill their names as operands. A type inside a class - literal (`Foo.class`) does not, since the literal is already one - operand, and neither does the inferred type `var`, which C# bills - nothing for either. #1561 made TypeScript's type-level operator - keywords operators, as their value-level siblings `as`, `typeof` - and `extends` already were: `satisfies`, `keyof`, `infer`, and the - `is` and `asserts` of a type predicate. -- **A generic list's angle brackets are one bracket pair.** The `<` +- **A declaration keyword is an operator.** Python's `class` is billed + like `def` since #1550, as Ruby, Kotlin, C# and PHP bill their + `class`. Java, Groovy, the JavaScript family, C, C++ and Objective-C + followed in #1552 with their type-declaration keywords: `class`, + `struct`, `union`, `enum`, `interface`, `record`, `trait`, + `@interface`, `@implementation` and `@protocol`. The `class` tail of + Java's `C.class` is an operand, not this keyword (see the class + literal below). Kotlin's `interface` followed in #1556. #1557 added + C++'s `namespace`, `template`, `typename`, `typedef` and `using` (C + and Objective-C bill `typedef` too), so `template ` and + `template ` each bill their keyword and score alike. It + also added TypeScript's `type`, `namespace`, `module`, `declare` and + `global`. #1561 made TypeScript's type-level operator keywords + operators, as their value-level siblings `as`, `typeof` and `extends` + already were: `satisfies`, `keyof`, `infer`, and the `is` and + `asserts` of a type predicate. #1564 made Kotlin's `typealias`, + `init`, `where`, `by`, `get` and `set` operators, as C# bills its + `get`, `set`, `init` and `where`. The `@get:` and `@set:` annotation + targets bill nothing, like `@field:`. The backing `field` in an + accessor is an operand, and the words used as names (`val by = 1`, + `x.get(i)`) stay operands. +- **A declaration modifier is an operator, keyed by its keyword and + never its wrapper node.** C#, Kotlin, Java, Groovy, TypeScript and + TSX, C, C++, Mozcpp and Objective-C bill every modifier once, as the + keyword and never as the node that wraps it: Kotlin's and Java's + `modifiers`, TypeScript's `accessibility_modifier` and + `override_modifier`, and the C family's `storage_class_specifier`, + `type_qualifier` and `virtual_specifier`. So the `const` of + `void f() const` and of `int * const p` each bill once. C# already + did; the others followed in #1558 (Kotlin), #1563 (TypeScript), #1565 + (Java and Groovy), #1578 (C, C++ and Mozcpp) and #1579 + (Objective-C). The exceptions: + - The same word used as a name stays an operand (`val open = 1`, + `let readonly = 1`, `obj.static`, `var sealed = 1`, + `int override = 1;`). Where Kotlin's grammar cannot parse the name + (`open = !open` at the start of a statement, `val suspend = 1`, a + parameter named `vararg`), the word bills nothing. + - The `static` of `import static` and `requires static` bills + nothing, as `import` does not. + - C++'s `auto` is a placeholder type and stays unbilled. Attributes, + the MS calling conventions, the Gecko `MOZ_*` macros and the Apple + SDK macros (`NS_INLINE`, `FOUNDATION_EXPORT`, `IBOutlet`) bill + nothing. + + The words each language bills: + + - Kotlin: the `enum` of `enum class`, `data`, `sealed`, the + visibility keywords, `open`, `override`, `suspend`, `reified`, + `out` and the rest. + - TypeScript and TSX: the member modifiers `public`, `private`, + `protected`, `override`, `readonly`, `abstract`, `accessor` and + `static`. JavaScript and Mozjs bill `static` too. Their grammars + lex `static get` followed by a newline as one `static get` token, + which bills as one operator, so that spelling scores one entry + where `static get y()` on one line bills `static` and `get`. + - Java and Groovy: the visibility keywords, `static`, `native`, + `strictfp`, `volatile`, `sealed` and `non-sealed`, where before + #1565 they billed only `abstract`, `final`, `transient`, + `synchronized` and `default`; plus the `permits` clause keyword + beside `extends` and `implements`. + - C, C++ and Mozcpp: every storage class, function specifier and + type qualifier: `static`, `extern`, `register`, `inline`, + `thread_local`, `const`, `volatile`, `restrict`, `_Atomic`, + `constexpr`, `alignas` and their vendor spellings (`__inline`, + `__forceinline`, `__restrict__`, `__thread`, `__volatile__`, + `__extension__`, …), plus C++'s `virtual`, `mutable`, `constinit`, + `consteval`, `explicit`, `friend`, the access labels, `override`, + `final` and `noexcept`. C bills its storage class `auto`. + `extern "C"` bills `extern` and keeps `"C"` an operand. + - Objective-C: C's list, plus the `@public`, `@private`, `@protected` + and `@package` ivar labels, the `@optional` and `@required` + protocol sections, the ownership qualifiers (`__weak`, `__strong`, + `__block`, …), the nullability qualifiers (`nullable`, + `_Nullable`, `__nullable`, …), `__kindof`, the `__covariant` and + `__contravariant` variance, and the `in`, `out`, `inout`, `bycopy`, + `byref` and `oneway` method-type qualifiers. A property attribute + (`nonatomic`, `copy`, `getter`) bills as its word; a `getter=` or + `setter=` value stays an operand. +- **A type name is an operand.** #1557 made every TypeScript type name + an operand (`x: Foo`, `interface I`, `Map`), as C#, Rust and + the C family already did. Java followed in #1560: `Foo f`, + `implements I`, `List` and each segment of `java.util.List` + bill their names as operands. The inferred type `var` does not, as + C# bills nothing for its own. Java's primitive types are operators + everywhere. Groovy, whose grammar parses a primitive type as a name, + bills it the same way since #1568: `int`, `boolean`, `void` and the + other primitives are operators keyed by their spelling, as a + declared type and as a class literal's receiver. Groovy reserves + these words, and they bill as operators; in the positions Groovy + nonetheless allows them as names (`x.int`, `[int: 1]`), they stay + operands. Java's `boolean` is listed as `boolean` rather than its + node kind `boolean_type`, so both languages report it alike. +- **A class literal bills its parts.** Since #1566 a Java class + literal bills its parts rather than one operand spelled by its whole + text, since + [JLS §15.8.2](https://docs.oracle.com/javase/specs/jls/se21/html/jls-15.html#jls-15.8.2) + defines it as an expression rather than a literal: in + `java.util.List.class` and `Foo[].class` each name and the `class` + tail are operands and each `.` and `[]` an operator, as Groovy's + `Foo.class` and Kotlin's and PHP's `Foo::class` bill theirs. The + `int` of `int.class` and the `void` of `void.class` stay operators, + as Java bills them everywhere. +- **Angle brackets that enclose types are one bracket pair.** The `<` and `>` around type arguments and type parameters count as one `<>` operator, billed at the `<` as `()` is at the `(`, and the `>` counts nothing (#1559). That covers `Vec`, `fn f`, a turbofish - `h::()`, `for<'a>` and `use` in Rust; C++'s + `h::()`, `for<'a>`, `use` and a qualified path's + `::` or `::` in Rust; C++'s `template ` and `std::vector`; and the type-argument and type-parameter lists of Java, Groovy, C#, Kotlin, TypeScript and - TSX. Objective-C has four generic lists: `NSArray`, - `id`, the protocol list in + TSX, plus C#'s `delegate*` pointer type and Kotlin's + `super` (#1567). Objective-C has four generic lists: + `NSArray`, `id`, the protocol list in `@interface A : NSObject `, and a type passed as a call argument, `f(NSArray)`. A comparison's `<` and `>` and a shift's `<<` and `>>` stay separate operators, and a C++ `>>` that closes two template lists @@ -761,12 +862,10 @@ tokens involved the same way they spell real operators: The exception is C#'s `>=` spelling, `F(a < b, c >= d)`, which the grammar parses as an assignment and which still bills `<>` and `=`. Go's generics use square brackets and already billed `[]`. Angle brackets around anything - other than a generic list still bill `<` and `>`: a Rust - `::` path, a C# `delegate*` pointer type and - a Kotlin `super`, as well as the JSX, Lua and Perl constructs - above. So does a Kotlin generic call with arguments, `id(a)`, - which tree-sitter-kotlin-ng parses as two comparisons (#1394), and a - generic inside a Rust macro call, `vec![Vec::::new()]`, whose + other than types still bill `<` and `>`: the JSX, Lua and Perl + constructs above. So does a Kotlin generic call with arguments, + `id(a)`, which tree-sitter-kotlin-ng parses as two comparisons + (#1394), and a generic inside a Rust macro call, `vec![Vec::::new()]`, whose token tree tree-sitter-rust leaves unparsed; a `>>` closing two lists there bills as the shift. Before #1559 each bracket billed under the comparison's own entry, @@ -816,7 +915,7 @@ tokens involved the same way they spell real operators: `operator[]`, `operator()`, `operator""_x` and `operator co_await`. The `operator` keyword is an operator in both languages, as is C++'s `co_await` in an expression. C++'s `[]` and `()` names are tokens of - their own, so `bca ops` lists `operator[]`'s `[]` beside a + their own, so `bca ops` lists `operator[]` beside a subscript's `[]` as a second operator, as Ruby does for `def [](i)` (#1296). C++'s coroutine keywords `co_return` and `co_yield` are operators as `return` is (#1547). @@ -835,7 +934,10 @@ tokens involved the same way they spell real operators: `&foo/2`, so the token inside it adds no operator; `&==/2` scores what `&foo/2` does. A captured `&&` / `||` / `and` / `or` is no decision either (#1534), but the remote call applies the operator, - so `Kernel.||(a, b)` is a cyclomatic decision as `a || b` is. This + so `Kernel.||(a, b)` scores what `a || b` does in cyclomatic, ABC and + cognitive alike, and a user module's `Foo.||(a, b)` in none (#1574). + `Kernel.and(a, b)` and `Kernel.or(a, b)` decide the same way, though + their names are plain identifiers with no operator token (#1582). This differs from a C# `operator <`, whose `<` has no wrapper and so stays the operator the rule above makes it. diff --git a/big-code-analysis-book/src/recipes/thresholds.md b/big-code-analysis-book/src/recipes/thresholds.md index 2b428797b..44f259402 100644 --- a/big-code-analysis-book/src/recipes/thresholds.md +++ b/big-code-analysis-book/src/recipes/thresholds.md @@ -93,46 +93,66 @@ tighten, not evidence that you should. | Language | `cognitive` | `cyclomatic` | `abc` | `halstead.effort` | `loc.ploc` | | --- | --- | --- | --- | --- | --- | -| Bash | 35 | 25 | 60 | 95000 | (thin sample) | -| C | 50 | 30 | 60 | 200000 | 3500 | -| C++ | 14 | 10 | 25 | 50000 | (thin sample) | -| C# | 4 | 4 | 13 | 9500 | 700 | +| Bash | 35 | 25 | 60 | 44000 | (thin sample) | +| C | 50 | 30 | 60 | 190000 | 3500 | +| C++ | 14 | 10 | 25 | 54000 | (thin sample) | +| C# | 4 | 4 | 13 | 44000 | 700 | | Elixir | 5 | 7 | 20 | 15000 | 1500 | -| Go | 25 | 17 | 45 | 120000 | 1500 | -| Groovy | 16 | 13 | 25 | 30000 | 450 | -| Java | 7 | 5 | 16 | 15000 | 800 | -| JavaScript | 20 | 15 | 35 | 55000 | (thin sample) | -| Kotlin | 10 | 8 | 20 | 20000 | 300 | -| Lua | 35 | 20 | 40 | 85000 | 800 | -| Objective-C | 18 | 12 | 35 | 60000 | (thin sample) | -| Perl | 35 | 25 | 45 | 140000 | 1500 | -| PHP | 10 | 9 | 25 | 50000 | 900 | -| Python | 20 | 13 | 25 | 25000 | 700 | -| Ruby | 7 | 6 | 19 | 10000 | 400 | -| Rust | 8 | 8 | 20 | 35000 | 900 | -| Tcl | 45 | 25 | 85 | 90000 | (thin sample) | -| TypeScript | 20 | 13 | 25 | 55000 | 450 | -| TSX | 14 | 10 | 65 | 95000 | 400 | +| Go | 25 | 17 | 45 | 89000 | 1500 | +| Groovy | 16 | 13 | 25 | 25000 | 450 | +| Java | 7 | 5 | 16 | 20000 | 800 | +| JavaScript | 20 | 15 | 35 | 170000 | (thin sample) | +| Kotlin | 10 | 8 | 20 | 18000 | 300 | +| Lua | 35 | 20 | 40 | 82000 | 800 | +| Objective-C | 18 | 12 | 35 | 77000 | (thin sample) | +| Perl | 35 | 25 | 45 | 41000 | 1500 | +| PHP | 10 | 9 | 25 | 26000 | 900 | +| Python | 20 | 13 | 25 | 42000 | 700 | +| Ruby | 7 | 6 | 19 | 5000 | 400 | +| Rust | 8 | 8 | 20 | 34000 | 900 | +| Tcl | 45 | 25 | 85 | 46000 | (thin sample) | +| TypeScript | 20 | 13 | 25 | 60000 | 450 | +| TSX | 14 | 10 | 65 | 130000 | 400 | "Thin sample" marks a language whose file count in the corpus is too small to derive a file-size figure from. The per-function figures for Bash, Tcl, JavaScript, and Objective-C rest on the smallest samples in the table and are the ones most worth re-deriving against your own code. +The `halstead.effort` column comes from a different measurement than the rest of the table. It was +re-measured on 2026-10-10, after the Halstead vocabulary changes in +[#1486](https://github.com/dekobon/big-code-analysis/issues/1486) through +[#1573](https://github.com/dekobon/big-code-analysis/issues/1573), on a rebuilt corpus of 41 +repositories that is comparable to the original but not identical to it. The C, C++, and +Objective-C rows were re-measured on 2026-10-15 to include the declaration modifiers of +[#1578](https://github.com/dekobon/big-code-analysis/issues/1578) and +[#1579](https://github.com/dekobon/big-code-analysis/issues/1579). The repository list and +commits are recorded in +[`docs/development/threshold_corpus.md`](https://github.com/dekobon/big-code-analysis/blob/main/docs/development/threshold_corpus.md). +Several rows moved by a factor of two to five, and almost all of that movement came from the change +of repositories, not from the metric: the same 41 repositories measured with the pre-change release +differ by at most half (Python) and mostly not at all. Treat any single row as a property of the +repositories that represent the language, which is one more reason to re-derive against your own. +In this column the thinnest samples are Objective-C, Perl, Bash, and C++, each under 2,500 +functions. + ### What to change, and why {#per-language-changes} Most languages need no override. The cases that do fall into three groups. **Procedural languages with large dispatch functions.** C, Tcl, Bash, Lua, Perl, and Go all run two -to three times the defaults. These languages lack exceptions or discourage them, so error handling -is inline branching, and they favour long `switch`-style dispatch over polymorphism. The default -`cognitive` limit flags 5% to 15% of their functions, against 3% in the median language. Raise -`cognitive` and `halstead.effort` first; those two carry most of the excess. +to three times the `cognitive` default and above the `cyclomatic` one. These languages lack +exceptions or discourage them, so error handling is inline branching, and they favour long +`switch`-style dispatch over polymorphism. The default `cognitive` limit flags 5% to 15% of their +functions, against 3% in the median language. Raise `cognitive` first. `halstead.effort` carries the +rest of the excess for C, Go, and Lua; Tcl, Bash, and Perl measure close to the default on it. In a single-language repository this is the whole `[thresholds]` table; in a mixed one it belongs under the language it describes, or it drags every other language up with it. ```toml -# Gating C. The same shape suits Tcl, Bash, Lua, Perl, and Go. +# Gating C. Tcl, Bash, Lua, Perl, and Go share its cognitive and +# cyclomatic shape, but not its effort: Go and Lua measure near 90000, +# and Tcl, Bash, and Perl near the default. [thresholds.lang.c] cognitive = 30 cyclomatic = 20 @@ -160,7 +180,8 @@ wmc = 40 ``` **Compact, heavily-abstracted languages.** Java, Ruby, Kotlin, C#, and Elixir sit far under the -defaults on every per-function metric. Their 97.5th percentile `cyclomatic` is 4 to 8. Gating them +defaults on every per-function metric except C#'s `halstead.effort`. Their 97.5th percentile +`cyclomatic` is 4 to 8. Gating them at 15 means the gate will effectively never fire, so tighten if you want it to do work. Read the C# row with care. Its distribution is dominated by property accessors, which `bca` counts diff --git a/big-code-analysis-cli/Cargo.toml b/big-code-analysis-cli/Cargo.toml index ff4a1fcfc..a3cf7b4ef 100644 --- a/big-code-analysis-cli/Cargo.toml +++ b/big-code-analysis-cli/Cargo.toml @@ -38,7 +38,7 @@ crossbeam = { version = "^0.8", features = ["crossbeam-channel"] } csv.workspace = true serde = { version = "^1.0", features = ["derive"] } serde_json = "^1.0" -serde_yaml = "^0.9" +serde-saphyr.workspace = true # `bca diff --since` materializes the tree at a git ref into an # auto-cleaning TempDir (#492), so tempfile is a runtime dependency, not # just a test helper. diff --git a/big-code-analysis-cli/src/default_thresholds.rs b/big-code-analysis-cli/src/default_thresholds.rs index 5c2bbb392..30a468879 100644 --- a/big-code-analysis-cli/src/default_thresholds.rs +++ b/big-code-analysis-cli/src/default_thresholds.rs @@ -15,6 +15,9 @@ //! and checked against a measured corpus: 43 real-world repositories //! across 20 languages (tests, vendored trees, and generated code //! excluded), 254k function spaces, 40k container spaces, 27k files. +//! `halstead.effort` alone was re-measured after the #1486-#1573 +//! vocabulary changes, on a rebuilt 41-repository corpus recorded in +//! `docs/development/threshold_corpus.md` (#1569). //! The design rule is that a default should flag genuine outliers — //! roughly 1-3% of in-scope spaces in the median language — because a //! gate that fires on a tenth of a codebase is a style rule, not a @@ -102,9 +105,10 @@ pub(crate) const DEFAULT_THRESHOLDS: &[DefaultThreshold] = &[ limit: 50_000, rationale: &[ "No published threshold exists for effort, so this one is purely", - "percentile-derived. It is also the most language-dependent limit", - "in the table — median-language p90 effort ranges from 2k (Java)", - "to 56k (C) — so it is the first one worth overriding per language.", + "percentile-derived: it fires on ~2% of functions in the median", + "language. It is also the most language-dependent limit in the", + "table — per-language p90 effort ranges from 3k (Ruby) to 64k", + "(C) — so it is the first one worth overriding per language.", ], }, DefaultThreshold { diff --git a/big-code-analysis-cli/src/formats.rs b/big-code-analysis-cli/src/formats.rs index 58929c986..b983e8a05 100644 --- a/big-code-analysis-cli/src/formats.rs +++ b/big-code-analysis-cli/src/formats.rs @@ -233,7 +233,7 @@ impl MetricsFormat { /// Bytes buffered in front of every output destination. /// /// The incremental serializers (`serde_json::to_writer`, -/// `serde_yaml::to_writer`, `ciborium::into_writer`) issue one write per +/// `serde_saphyr::to_io_writer`, `ciborium::into_writer`) issue one write per /// structural token, and both a raw `File` and a `StdoutLock` pass each /// one straight to the kernel: a 12 MB aggregate document measured /// 4.76 million `write(2)` calls, ~2.6 bytes apiece. 64 KiB is large @@ -339,7 +339,7 @@ pub(crate) fn dump_aggregate( output: &Path, pretty: bool, ) -> std::io::Result<()> { - write_buffered_file(output, |w| match format { + write_buffered_file(output, |mut w| match format { GenericFormat::Json => { if pretty { serde_json::to_writer_pretty(w, &items).map_err(ser_err) @@ -362,7 +362,7 @@ pub(crate) fn dump_aggregate( }; w.write_all(text.as_bytes()) } - GenericFormat::Yaml => serde_yaml::to_writer(w, &items).map_err(ser_err), + GenericFormat::Yaml => serde_saphyr::to_io_writer(&mut w, &items).map_err(ser_err), GenericFormat::Cbor => ciborium::into_writer(&items, w).map_err(ser_err), }) } @@ -645,7 +645,7 @@ struct Yaml; impl RenderDocument for Yaml { fn format(content: T) -> std::io::Result { - serde_yaml::to_string(&content).map_err(ser_err) + serde_saphyr::to_string(&content).map_err(ser_err) } } @@ -657,8 +657,8 @@ impl WriteFile for Yaml { path: &Path, output_path: &Path, ) -> std::io::Result<()> { - Self::with_file(path, output_path, |w| { - serde_yaml::to_writer(w, &content).map_err(ser_err) + Self::with_file(path, output_path, |mut w| { + serde_saphyr::to_io_writer(&mut w, &content).map_err(ser_err) }) } } @@ -736,7 +736,7 @@ pub(crate) fn write_json( /// Serialize `value` as YAML and write it through [`write_text`]. See /// [`write_json`] for why these live here. pub(crate) fn write_yaml(value: &T, output: Option<&PathBuf>) -> std::io::Result<()> { - let yaml = serde_yaml::to_string(value).map_err(std::io::Error::other)?; + let yaml = serde_saphyr::to_string(value).map_err(std::io::Error::other)?; write_text(&yaml, output) } @@ -1117,10 +1117,36 @@ mod tests { let yaml = dir.path().join("agg.yaml"); dump_aggregate(GenericFormat::Yaml, &expected, &yaml, false).expect("yaml"); let got: Vec = - serde_yaml::from_str(&std::fs::read_to_string(&yaml).expect("read")).expect("parse"); + serde_saphyr::from_str(&std::fs::read_to_string(&yaml).expect("read")).expect("parse"); assert_eq!(got, expected); } + #[test] + fn yaml_preserves_ambiguous_strings_and_full_precision() { + let expected = [ + "true", + "null", + "01", + "1e3", + "a: b", + "*alias", + "# comment", + "", + "café\nline\n", + ] + .map(|name| Item { + name: name.to_owned(), + value: 0.123_456_789_012_345_68, + }); + let text = Yaml::format(&expected).expect("render YAML"); + let parsed: Vec = serde_saphyr::from_str(&text).expect("parse YAML"); + assert_eq!(parsed, expected); + let directory = tempfile::tempdir().expect("tempdir"); + let path = directory.path().join("aggregate.yaml"); + dump_aggregate(GenericFormat::Yaml, &expected, &path, false).expect("write YAML"); + assert_eq!(std::fs::read_to_string(path).expect("read YAML"), text); + } + /// The `pretty` flag has to reach the serializer. Both spellings /// parse back to the same value, so a round-trip alone cannot tell /// them apart — the discriminating check is that exactly one of them @@ -1163,7 +1189,7 @@ mod tests { let yaml = dir.path().join("out.yaml"); write_yaml(&value, Some(&yaml)).expect("yaml"); assert_eq!( - serde_yaml::from_str::>(&std::fs::read_to_string(&yaml).expect("read")) + serde_saphyr::from_str::>(&std::fs::read_to_string(&yaml).expect("read")) .expect("parse"), value ); diff --git a/big-code-analysis-cli/tests/output/format_smoke.rs b/big-code-analysis-cli/tests/output/format_smoke.rs index 0b7e7e699..40e8bf82c 100644 --- a/big-code-analysis-cli/tests/output/format_smoke.rs +++ b/big-code-analysis-cli/tests/output/format_smoke.rs @@ -461,11 +461,11 @@ fn cli_metrics_yaml_round_trips() { let fixture = write_rust_fixture(&dir); let out = run_metrics(&dir, "yaml", &fixture); - let doc: serde_yaml::Value = serde_yaml::from_str(&out).expect("metrics YAML parses"); - let map = doc.as_mapping().expect("YAML root is a mapping"); + let doc: serde_json::Value = serde_saphyr::from_str(&out).expect("metrics YAML parses"); + let map = doc.as_object().expect("YAML root is a mapping"); for key in STRUCTURAL_KEYS { assert!( - map.contains_key(serde_yaml::Value::String((*key).to_string())), + map.contains_key(*key), "YAML output missing top-level key {key:?}; doc was:\n{out}", ); } @@ -555,7 +555,7 @@ fn cli_metrics_all_formats_agree_on_metric() { let json = json_metric(&dir, &fixture, CYCLOMATIC_KEY); let yaml_out = run_metrics(&dir, "yaml", &fixture); - let yaml: serde_yaml::Value = serde_yaml::from_str(&yaml_out).expect("YAML parses"); + let yaml: serde_json::Value = serde_saphyr::from_str(&yaml_out).expect("YAML parses"); let yaml = yaml["metrics"][CYCLOMATIC_KEY.0][CYCLOMATIC_KEY.1] .as_f64() .expect("YAML metric is numeric"); diff --git a/big-code-analysis-cli/tests/output/output_unification.rs b/big-code-analysis-cli/tests/output/output_unification.rs index 2434a9551..241868bc0 100644 --- a/big-code-analysis-cli/tests/output/output_unification.rs +++ b/big-code-analysis-cli/tests/output/output_unification.rs @@ -529,7 +529,7 @@ fn per_file_documents_survive_spanning_several_output_buffers() { v.to_string() } "yaml" => { - let v: serde_yaml::Value = serde_yaml::from_slice(&bytes) + let v: serde_json::Value = serde_saphyr::from_slice(&bytes) .unwrap_or_else(|e| panic!("{format}: truncated or invalid document: {e}")); serde_json::to_string(&v).unwrap() } diff --git a/big-code-analysis-cli/tests/vcs/vcs_trend.rs b/big-code-analysis-cli/tests/vcs/vcs_trend.rs index d1f3c5051..502c91fea 100644 --- a/big-code-analysis-cli/tests/vcs/vcs_trend.rs +++ b/big-code-analysis-cli/tests/vcs/vcs_trend.rs @@ -152,17 +152,17 @@ fn yaml_format_emits_valid_document() { .output() .expect("run bca vcs trend yaml"); assert!(output.status.success()); - let doc: serde_yaml::Value = - serde_yaml::from_slice(&output.stdout).expect("trend output is valid YAML"); + let doc: serde_json::Value = + serde_saphyr::from_slice(&output.stdout).expect("trend output is valid YAML"); assert_eq!(doc["trend_schema_version"], 1); // Non-vacuous structural checks: the point grid and file map are // present with the requested shape. assert_eq!( - doc["as_of_points"].as_sequence().map(Vec::len), + doc["as_of_points"].as_array().map(Vec::len), Some(2), "two requested points appear in the YAML" ); - assert!(doc["files"].is_mapping(), "files is a path-keyed mapping"); + assert!(doc["files"].is_object(), "files is a path-keyed mapping"); } /// A history-tuning flag (`--as-of`) is accepted in the *subcommand* diff --git a/big-code-analysis-web/Cargo.toml b/big-code-analysis-web/Cargo.toml index 79272c234..0f786c107 100644 --- a/big-code-analysis-web/Cargo.toml +++ b/big-code-analysis-web/Cargo.toml @@ -46,7 +46,7 @@ serde_json = "^1.0" # same value. Versions mirror the `big-code-analysis-cli` pins; both # crates already sit in the workspace graph, so this adds no new resolved # dependency. -serde_yaml = "^0.9" +serde-saphyr.workspace = true ciborium = "^0.2" tokio = { version = "1", features = ["rt", "sync", "time"] } tracing = "^0.1" diff --git a/big-code-analysis-web/src/web/negotiate.rs b/big-code-analysis-web/src/web/negotiate.rs index bd551f340..7118ac3c9 100644 --- a/big-code-analysis-web/src/web/negotiate.rs +++ b/big-code-analysis-web/src/web/negotiate.rs @@ -57,7 +57,7 @@ impl Format { pub(super) fn encode(self, value: &T) -> Result, ()> { match self { Format::Json => serde_json::to_vec(value).map_err(|_| ()), - Format::Yaml => serde_yaml::to_string(value) + Format::Yaml => serde_saphyr::to_string(value) .map(String::into_bytes) .map_err(|_| ()), Format::Cbor => { @@ -278,8 +278,8 @@ mod tests { assert_eq!(parsed["n"], serde_json::json!(7)); let yaml = Format::Yaml.encode(&value).unwrap(); - let parsed: serde_yaml::Value = serde_yaml::from_slice(&yaml).unwrap(); - assert_eq!(parsed["n"], serde_yaml::Value::from(7)); + let parsed: serde_json::Value = serde_saphyr::from_slice(&yaml).unwrap(); + assert_eq!(parsed["n"], serde_json::Value::from(7)); let cbor = Format::Cbor.encode(&value).unwrap(); let parsed: ciborium::Value = ciborium::from_reader(cbor.as_slice()).unwrap(); diff --git a/big-code-analysis-web/src/web/server_tests.rs b/big-code-analysis-web/src/web/server_tests.rs index 1bb9c1d37..094950eff 100644 --- a/big-code-analysis-web/src/web/server_tests.rs +++ b/big-code-analysis-web/src/web/server_tests.rs @@ -4237,11 +4237,11 @@ async fn test_web_metrics_yaml_accept_emits_yaml() { let body = test::read_body(resp).await; // The body must parse as YAML and carry the same fields the JSON body // would, proving the response is genuinely YAML and not JSON mislabelled. - let parsed: serde_yaml::Value = serde_yaml::from_slice(&body).unwrap(); - assert_eq!(parsed["id"], serde_yaml::Value::from("neg-657")); - assert_eq!(parsed["language"], serde_yaml::Value::from("python")); + let parsed: Value = serde_saphyr::from_slice(&body).unwrap(); + assert_eq!(parsed["id"], Value::from("neg-657")); + assert_eq!(parsed["language"], Value::from("python")); // YAML is a JSON superset, so a successful YAML parse alone does not rule - // out a JSON body mislabelled as YAML. serde_yaml emits this nested map + // out a JSON body mislabelled as YAML. serde_saphyr emits this nested map // in block style (`id: neg-657\n...`), which is not valid JSON — so a // failed JSON parse is positive proof the bytes are real YAML. assert!( @@ -4286,10 +4286,8 @@ async fn test_web_metrics_yaml_matches_json_payload() { let yaml_resp = test::call_service(&app, metrics_request(Some("application/yaml")).to_request()).await; let yaml_body = test::read_body(yaml_resp).await; - let yaml: serde_yaml::Value = serde_yaml::from_slice(&yaml_body).unwrap(); - // Re-serialize the YAML value to JSON and compare structurally. - let yaml_as_json: Value = serde_json::to_value(&yaml).unwrap(); - assert_eq!(json, yaml_as_json); + let yaml: Value = serde_saphyr::from_slice(&yaml_body).unwrap(); + assert_eq!(json, yaml); } #[actix_rt::test] @@ -4353,8 +4351,8 @@ async fn test_web_ast_yaml_accept_emits_yaml() { assert_eq!(resp.status(), StatusCode::OK); assert_eq!(content_type(&resp), "application/yaml"); let body = test::read_body(resp).await; - let parsed: serde_yaml::Value = serde_yaml::from_slice(&body).unwrap(); - assert_eq!(parsed["id"], serde_yaml::Value::from("ct-515")); + let parsed: Value = serde_saphyr::from_slice(&body).unwrap(); + assert_eq!(parsed["id"], Value::from("ct-515")); } #[actix_rt::test] diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index 50a7e0886..17f87c38d 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -58,8 +58,9 @@ git submodule update --init --recursive ## The complexity-class gate {#complexity-gate} Each probe pairs a generated source shape with the workload that -exercises one hot path — a metric selection through `Ast::metrics`, or -the operator/operand walk behind `Ast::ops`. The shape is rendered at three doubling +exercises one hot path — a metric selection through `Ast::metrics`, +the operator/operand walk behind `Ast::ops`, or a `--type` filter +through `Ast::count`. The shape is rendered at three doubling sizes, every (probe, size) cell is measured once per round with the visit order rotated between rounds, and the reported figure is the slope of `ln(time)` against `ln(size)`. A linear walk sits near 1.0; @@ -155,12 +156,25 @@ Read it as follows. | `abc/wide-let-declarators` | JavaScript | width | ABC's structural `const` check per `let` declarator (#1277) | linear | | `loc/wide-cfg-test-mod` | Rust | width | the `exclude_tests` prune arm's row bookkeeping (#1417) | linear | | `halstead/perl-dash-keys` | Perl | width | the dash-key workaround's `=>` lookup per key (#1545) | linear | +| `abc/perl-bareword-keys` | Perl | width | the bareword-key workaround's `=>` lookup per key (#1572) | linear | +| `halstead/kotlin-short-interpolations` | Kotlin | width | the `$name` test's previous-token lookup in one string (#1584) | linear | | `halstead/kotlin-noinline-params` | Kotlin | width | the modifier recovery guard's next-token lookup (#1558) | linear | | `abc/perl-pattern-list` | Perl | width | ABC's pattern-role climb through `( … )` (#1467) | linear | +| `cognitive/elixir-mixed-arguments` | Elixir | width | the keyword-section lookup per pair and per keyword-form clause (#1574) | linear | | `cyclomatic/cpp-plus-chain` | C++ | depth | shape control for the three rows below | linear | | `cyclomatic/cpp-and-chain` | C++ | depth | the `&&` / `||` applied-operator gate (#1533) | linear | | `cognitive/cpp-and-chain` | C++ | depth | the boolean-sequence arm on the same chain (#1533) | linear | | `abc/cpp-and-chain` | C++ | depth | ABC's chain arm on the same chain (#1533) | linear | +| `abc/perl-or-chain` | Perl | depth | the misparse check's spine climb (#1273) | linear | +| `abc/perl-misparsed-comparison` | Perl | depth | the same climb on a chain the misparse absorbed (#1273) | linear | +| `abc/perl-and-statements` | Perl | width | the misparse check's operator lookup in a block (#1273) | linear | +| `abc/perl-error-tail` | Perl | width | the binding, bareword-key and misparse lookups under a flat `ERROR` root (#1540, #1584) | linear | +| `halstead/perl-error-tail-keys` | Perl | width | the key and sign lookups under the same `ERROR` (#1584) | linear | +| `nexits/perl-error-tail-keys` | Perl | width | nexits' `die` / `exit` key lookup under the same `ERROR` (#1585) | linear | +| `count/perl-error-tail-calls` | Perl | width | the `call` filter's bareword-key lookup under the same `ERROR`, on the `find` / `count` walk (#1580) | linear | +| `cognitive/bash-test-connective-run` | Bash | depth | the comparison-run end of a `[ … -o … ]` test (#1536) | linear | +| `abc/bash-arith-assign-chain` | Bash | depth | ABC's `=` comparison test on `(( a = … ))` (#1536) | linear | +| `abc/bash-eq-and-test` | Bash | depth | the same test on a `[[ … = … && … ]]` run (#1536) | linear | Four of these were quadratic when the harness landed, and they shared one cause: `tree_sitter` stores no parent pointer, so `Node::parent` @@ -364,6 +378,63 @@ depth 4 000 — while `cyclomatic/cpp-plus-chain`, the same chain through metric has its own probe because each reaches the chain through its own arm, and a climb added to one of them would leave the other two flat. +The seven Perl and Bash probes guard walks that went quadratic during +the #1273, #1536 and #1540 work and were caught only by a hand-timed +check in review ([#1575][perl-bash-probes]). Each was watched failing +with its pre-fix code put back behind a local patch, one walk at a +time: + +| Walk put back | Probe that fails | Clean | Regressed | Top rung | +|---|---|---|---|---| +| every Perl spine node climbs | `abc/perl-or-chain` | 1.00 | 2.00 | 8.8 ms → 2.33 s | +| (same patch) | `abc/perl-misparsed-comparison` | 0.99 | 2.01 | 8.8 ms → 2.35 s | +| the stop parent scanned from its start | `abc/perl-and-statements` | 1.08 | 2.01 | 3.7 ms → 306 ms | +| any parent read as a binding | `abc/perl-error-tail` | 1.19 | 2.05 | 2.9 ms → 492 ms | +| the Bash comparison run climbed per node | `cognitive/bash-test-connective-run` | 0.98 | 2.06 | 6.5 ms → 1.99 s | +| every Bash `=` climbs to its test | `abc/bash-arith-assign-chain` | 1.00 | 1.97 | 0.8 ms → 72.5 ms | +| (same patch) | `abc/bash-eq-and-test` | 1.02 | 1.91 | 2.9 ms → 75.2 ms | + +In each run only the named probes moved, and every other probe stayed +inside its bound. + +Removing only the kind gate in front of the Perl lookup moved nothing: +`abc/perl-and-statements` held at 1.19. On a block that gate is not +what keeps the walk linear, because the lookup seeks the child by byte. +That seek is `O(log n)` only where the grammar stores the children as +a balanced repeat. An `ERROR` that recovery built by skipping tokens is +one flat array, which the seek scans from its start, and no tree-sitter +call reaches a child of one in better than its index +([#1584][flat-error]). So the metric walk and the `ops` walk record the +children of every node wider than 64 as they push them, in a +`SiblingCache` they lend through `Ancestors`, and +`Ancestors::siblings_after` / `siblings_before` binary-search that +array instead of seeking. `Ancestors::previous_sibling` reads it too: +it scanned its parent from the start, which made a valid Kotlin string +of `$name` interpolations quadratic (11.5 s for 8 000 names). With +`Ancestors::lookup` patched to ignore the cache, these four probes go +over bound, as does `count/perl-error-tail-calls` below, whose walk +reads the same cache: + +| Walk put back | Probe that fails | Clean | Regressed | Top rung | +|---|---|---|---|---| +| sibling lookups seek in the flat `ERROR` | `abc/perl-error-tail` | 1.08 | 2.10 | 8.8 ms → 326 ms | +| (same patch) | `halstead/perl-error-tail-keys` | 1.04 | 1.74 | 14.5 ms → 96.0 ms | +| (same patch) | `halstead/kotlin-short-interpolations` | 0.96 | 2.02 | 1.1 ms → 589 ms | +| (same patch) | `nexits/perl-error-tail-keys` | 1.11 | 1.87 | 5.5 ms → 127 ms | + +The walk `bca find` and `bca count` share, `act_on_node`, keeps the same +chain and lent no cache until Perl's `call` filter started asking the +bareword-key test of every bareword ([#1580][find-call]). It records +one now, and `count/perl-error-tail-calls` prices it: + +| Walk put back | Probe that fails | Clean | Regressed | Top rung | +|---|---|---|---|---| +| `act_on_node` lends no cache | `count/perl-error-tail-calls` | 0.88 | 1.89 | 2.8 ms → 84.5 ms | + +These four are the probes whose shape is meant not to parse: +`RECOVERY_PROBES` in `shapes.rs` exempts them from the parse check and +asserts that each root is still an `ERROR`. + Treat the linear bounds above as covering the walk's ancestor *chain* threading, not every `O(depth)` lookup in the crate. @@ -460,6 +531,9 @@ a walk's chain bookkeeping, not just around a change to its cost. The [cfg-predicate]: https://github.com/dekobon/big-code-analysis/issues/1105 [attribute-row]: https://github.com/dekobon/big-code-analysis/issues/1431 [space-merge]: https://github.com/dekobon/big-code-analysis/issues/1106 +[perl-bash-probes]: https://github.com/dekobon/big-code-analysis/issues/1575 +[flat-error]: https://github.com/dekobon/big-code-analysis/issues/1584 +[find-call]: https://github.com/dekobon/big-code-analysis/issues/1580 The eleven control probes are what make the other readings mean something. @@ -523,7 +597,8 @@ alone. Add a `Probe` to `PROBES` in `big-code-analysis-bench/src/shapes.rs`. The unit tests in that module enforce what a probe has to satisfy: bytes affine in the size parameter, no parse errors at the smallest -declared size, a non-zero workload reading there, sizes that double, +declared size (or, for a probe listed in `RECOVERY_PROBES`, an `ERROR` +root), a non-zero workload reading there, sizes that double, and the invariant its `axis` selects — AST levels growing with the size on `Axis::Depth`, children growing while AST depth holds still on `Axis::Width`. Set `max_exponent` from a measurement on an idle host, diff --git a/docs/development/lessons_learned.md b/docs/development/lessons_learned.md index f8a3e1e1d..b22b4611e 100644 --- a/docs/development/lessons_learned.md +++ b/docs/development/lessons_learned.md @@ -139,6 +139,8 @@ number and the higher number stays as a redirect. | [93](#93-a-gate-that-reads-a-typed-accessor-is-invisible-to-the-front-ends-that-read-the-wire) | A gate that reads a typed accessor is invisible to the front-ends that read the wire | | [94](#94-a-sweep-that-finds-nothing-only-rules-out-what-its-predicate-can-express) | A sweep that finds nothing only rules out what its predicate can express | | [95](#95-error-recovery-inserts-tokens-that-is_missing-does-not-flag) | Skip invented tokens by width, not `is_missing()` — valid code triggers it too | +| [96](#96-a-tree-sitter-child-seek-is-olog-n-only-through-a-grammar-repetition) | A tree-sitter child seek is O(log n) only through a grammar repetition — the walk is the O(1) source of siblings | +| [97](#97-a-beforeafter-on-a-rebuilt-corpus-measures-the-corpus) | A before/after on a rebuilt corpus measures the corpus — run the old binary on it too | --- @@ -2112,7 +2114,11 @@ slot from positional to field addressing, dump the AST for *each* spelling of the construct before choosing the fixture: two spellings that look interchangeable can put the `;` at different indices, and an empty-slot regression test written against the wrong one passes -vacuously under the old code. +vacuously under the old code. The same rule holds upward: a fixed-depth +parent climb (`up.nth(2)`) is positional addressing of an ancestor. +Climb by role, stepping over at most one optional wrapper, and verify +the position from the siblings along the chain you climbed rather than +by re-reading the construct from its front. Each language's grammar makes a different choice about which slots are optional, so the bug is per-language and per-statement-kind, not @@ -2148,6 +2154,18 @@ deeper, every `*_inspect_container` but Tcl's and iRules' still unwraps with `node.child(1)`, so `if ((/* n */ a))` scores zero there — #1181's defect inside the wrapper, left open. +**Elixir's section lookup climbed exactly two parents** (#1574). A +trailing bracketed keyword list `[do: …, else: …]` inserts a `list` +between `keywords` and `arguments`, so the climb from a pair landed on +`arguments` instead of the Call and answered "no owner" plausibly: +`with` lost its modified-cyclomatic `else` tier, `for … reduce:` scored +its accumulator head as a decision, `cond` lost its free `true ->`, and +keyword-form `if` / `unless` lost ABC's `else` condition. The fix climbs +by role over at most one bracketing `list` and decides from the chain's +siblings: no operand may follow the argument, and in a bracketed list no +operand may precede `keywords`. That is also what rejects a non-trailing +or nested list. + --- ## 54. A "no-op regen" must be proven by an actual regen + diff @@ -2258,7 +2276,10 @@ access"). A shape-only proxy passes every test whose fixtures happen to use the expected identity and silently mis-handles every other receiver; if you must approximate, document it as an under-approximation and prove the boundary with a fixture using a *foreign* receiver. Do not reach for -`to_string_lossy()` here (lesson 43). +`to_string_lossy()` here (lesson 43). When a grammar folds a reserved +word into the name kind, gate by text in every position the word can +occupy and exclude only the positions the language specification allows +it as a name — not the one kind the issue happens to name. "An `Attribute` whose first child is an `Identifier`" *looks* like `self.x`, but it is equally `db.x` and `logger.x`. The shape answers "is @@ -2277,6 +2298,17 @@ inflated NPA directly. The fix matches receiver bytes against the dedup key, and separates `self.f.g = 1` (an attribute on `self.f`) from `self.g = 1` — a distinction the structural proxy could never make. +**Groovy billed its primitive types as operands** (#1568). The grammar +has no primitive node: `int` is a `type_identifier` in a declaration but +a plain `identifier` in `int.class` and in an uninitialised `int t`, so +the issue's proposed `type_identifier`-only gate would have left half +the shapes diverging from Java. The fix classifies the nine spellings by +text wherever they appear and excludes the name positions the Groovy +spec permits — after a member-access operator (`x.int`, `x?.int`, +`x*.int`, `x::int`) and as a map or named-argument label (`[int: 1]`, +`f(int: 1)`) — pinned by a Java-twin test on shared declarations and by +the class-literal rows in one shared expectation table. + --- ## 58. A wrapper-node + keyword leaf is one operator, not two @@ -2538,7 +2570,10 @@ increment — so its default really is redundant — before suppressing, and confirm the shared node kind is not also serving a chain construct where the arm is a legitimate +1. See [`grammar-dispatch.md` §8](../../.claude/rules/grammar-dispatch.md) and -lesson 11 for the cross-metric agreement rule. +lesson 11 for the cross-metric agreement rule. A construct can offer two +candidate defaults; exactly one is free. Decide by counting reachable +paths, and write the fixture that carries both arms — one that carries +either is satisfied by the wrong rule. **Ruby `case`-else is +0 but `begin`/`rescue`-else is +1** (#451, `30a435ae`). The shared `R::Else` node appears under three constructs. A @@ -2551,6 +2586,16 @@ plan proposed suppressing rescue-else too**; doing so would have introduced a *fresh* divergence from Python. The fix gates only on `parent ∈ {Case, CaseMatch}`. +**Elixir `receive` with both `_ ->` and `after` had two free arms** +(#1581, caught by the whole-branch review). #1581 made the `after` +timeout clause free as "receive's default arm, the role `_ ->` plays in +a `case`" — but `_ ->` stayed free under `receive` too. `receive do +{:a, y} -> y; _ -> 0 after 10 -> :t end` has three reachable paths and +scored one decision, the same as a two-path `case`. The per-issue +review and `audit-tests` both passed, because every fixture had one arm +or the other and never both. The rule is now one free arm: the catch-all +if the `do` section has one, otherwise `after`. + --- ## 65. Removing a node kind from `is_func` / `is_func_space` zeroes its childless variant @@ -3188,7 +3233,12 @@ wrong answers rather than a failure, and a parity test written against a step `O(1)`, not the climb: a predicate that walks "until the first ancestor that is not X" is still `O(depth)` per node wherever X can nest without bound, so propagate the *answer* instead, and add a -`bench-scaling` probe of that shape before trusting the gate. +`bench-scaling` probe of that shape before trusting the gate. And +before deriving a chain for a predicate that lacks one, check whether +its caller's walk already carries it: the `find` / `count` `call` +filter had `ancestors` in hand all along, so a `_with_code` spelling +that forwards by default was cheaper, and linear, where a new +`Node::parent` climb would have been neither (#1580). `tree_sitter` stores no parent pointer — `Node::parent` restarts at the root and descends — so any predicate asking a node for an ancestor is @@ -3898,7 +3948,12 @@ what the author wrote must key on **width** (`start_byte() == end_byte()`), not on `Node::is_missing()`. Prove it with one malformed fixture per grammar the guard covers. Do not assume the trap needs invalid input: grammars cannot parse every valid program, and they -recover from the gap the same way. +recover from the gap the same way. The inverse trap is a *real* token +that recovery wraps in an `ERROR` it marks as an extra: a sibling helper +written to step over comments steps over the token too. Skip +`is_extra() && !is_error()` only, and prove it with a fixture whose +recovery puts an `ERROR` between the subject and the token the helper +looks for. When tree-sitter recovers from a parse error, it can insert a token the source lacks, such as a `;`, a `)` or an identifier. The inserted token @@ -3926,4 +3981,84 @@ contain one. valid trees contain zero-width nodes too, such as an empty Bash heredoc body. +**A stray `,` before `=>` hid a real token inside an extra** (#1572). +`(name, => 1)` is valid Perl — consecutive commas are legal, and `=>` +quotes only the word immediately to its left, so `name` is a call — but +the grammar recovers by wrapping the `,` in an `ERROR` flagged +`is_extra()`. The key-position predicate's `next_non_extra`, written to +look past comments, skipped it and quoted `name` as a hash key. The +fix's review caught it; the helper now passes over comments only, as +the rule `perl_not_is_key` beside it already did. + +--- + +## 96. A tree-sitter child seek is O(log n) only through a grammar repetition + +**Lesson:** Answer "which sibling sits next to this child" from the walk +that pushed the children, never from a tree-sitter seek on the parent. +`goto_first_child_for_byte`, `Node::child(i)`, `next_sibling` and +`goto_previous_sibling` are O(log n) only when the parent's children are +stored as a grammar `repeat()`'s balanced hidden subtree. Over a flat +child array — every `ERROR` that recovery builds by skipping tokens — +each is O(index), and a per-child sibling question is quadratic. Record a +wide node's children when the walk pushes them, lend the record through +the ancestor chain, and gate the shape with a `bench-scaling` probe whose +body really flattens: plain repetition folds into nested +`binary_expression`s and keeps its repeat structure, so a probe needs a +stray `{` or `$d` per unit before it measures the flat case. + +The cost is invisible on the inputs anyone tests. Valid code and an +unclosed list both keep the repetition's hidden levels, so a seek that +is documented O(log n) is O(log n) there, and a timing probe written +from the same shapes fits linear. Only an `ERROR` holding its children +as one flat array exposes the scan — and real files produce one: a +Perl debugger script whose tail fails to parse has 30,798 direct +children under its root. + +**Perl ABC and Halstead were quadratic under a flat `ERROR`** (#1584, +found while writing #1575's probes). `children_after` / +`children_before` from #1545 and #1558 carried an O(log n) claim that +held for the unclosed-list probe written beside them, because that list +kept its repeat structure. With a `{` per body the root became a flat +`ERROR` and the bareword-key and misparse lookups ran 4.07× per +doubling, 8.99 s at 32,000 keys. Halstead and `ops` hit the same scan +through the dash-key and `not`-key helpers, and +`Ancestors::previous_sibling` made Kotlin Halstead quadratic on a +*valid* string of 8,000 `$a` interpolations (11.47 s). The fix records +each node with more than `WIDE_PARENT` children into a per-depth +`SiblingCache` as the metric, `ops` and `find`/`count` walks push them, +and every lookup falls back to the old seek on a miss; five probes trip +at 1.7–2.1 with the cache ignored and fit 0.9–1.1 with it. + +--- + +## 97. A before/after on a rebuilt corpus measures the corpus + +**Lesson:** When re-deriving a calibrated default after a metric +definition changes, measure the rebuilt corpus with the previous release +binary as well as the new one, and attribute each row's movement to the +metric or to the sample before touching a limit or a published table. +Record the corpus — repositories, commit SHAs, exclusion globs — in the +repository, so the next re-derivation has an identical sample to run +both binaries against instead of a comparable one. + +A percentile of a sampled corpus moves with the sample. A metric change +and a sample change present identically — the per-language rows moved — +and the issue that asks for the re-derivation presupposes the first +cause, because the metric change is what prompted it. Without the +old-binary run on the same files there is nothing to separate the two, +and the stale-looking rows read as drift to correct. + +**The `halstead.effort` default survived a re-derivation that looked +like a 5× drift** (#1569). #1140's 43-repository corpus had never been +recorded, so a comparable 41-repository one was rebuilt. Against the +book's rows it moved 2–5× (C# 4.6×), and the issue's own evidence called +Java's rationale "off by about 5×". Run with the v2.3.0 binary on the +identical rebuilt files, the vocabulary changes moved p97.5 by at most +47% (Python) and the median-language flag rate from 2.23% to 2.20%: +almost all of the movement was the change of repositories. The default +stayed at 50,000, the book's column was regenerated from the new +corpus, and the corpus is recorded in +`docs/development/threshold_corpus.md`. + --- diff --git a/docs/development/threshold_corpus.md b/docs/development/threshold_corpus.md new file mode 100644 index 000000000..daa801b98 --- /dev/null +++ b/docs/development/threshold_corpus.md @@ -0,0 +1,130 @@ +# Threshold calibration corpus + +This file records the corpus behind the `halstead.effort` default in +[`big-code-analysis-cli/src/default_thresholds.rs`](../../big-code-analysis-cli/src/default_thresholds.rs) +and the `halstead.effort` column of the book's per-language table +([Choosing thresholds](../../big-code-analysis-book/src/recipes/thresholds.md#per-language)). +The other defaults and columns still rest on +[#1140](https://github.com/dekobon/big-code-analysis/issues/1140)'s 2026-07-31 corpus: 43 repositories +across 20 languages, whose repository list was never recorded. + +The corpus below is a **rebuilt** one, made for +[#1569](https://github.com/dekobon/big-code-analysis/issues/1569) after the Halstead vocabulary +changes (#1486 through #1573). It covers the same 20 languages with the same sizing rule and the same kinds of +exclusions, so it is comparable to #1140's, but it is not the same set of repositories. + +## Method + +- Measured 2026-10-10 with a release `bca` built from `fix/batch-2026-10-10` at `18c28cac`. The C, + C++, and Objective-C rows were re-measured on 2026-10-15 at `d361bc32`, the full branch, because + three later changes bill more C-family declaration modifiers: #1578 (C, C++, Mozcpp), #1579 + (Objective-C), and `__volatile__`. No other language's vocabulary changed after `18c28cac`, and the + C#, Elixir, Perl, and Ruby rows reproduce exactly at `d361bc32`. The same corpus was also measured + with the `v2.3.0` release, which predates all of these changes, to separate the metric change from + the corpus change. +- Each repository was cloned with `git clone --depth 1` at its default branch. +- `bca metrics --no-config -O json -p ` ran once per repository. The `-I` globs restricted it to + the language's own extensions: `*.sh` (Bash-it also `*.bash`), `*.c`, `*.cpp *.cc *.h *.hpp`, `*.cs`, + `*.ex`, `*.go`, `*.groovy`, `*.java`, `*.js *.mjs`, `*.kt`, `*.lua`, `*.m`, `*.pm *.pl`, `*.php`, + `*.py`, `*.rb`, `*.rs`, `*.tcl`, `*.ts`, `*.tsx`. C counts `.c` only, because `bca` parses `.h` with + the C++ grammar. +- The run excluded test, vendored, generated, and documentation-demo trees with these `-X` globs: + `**/test/**`, `**/tests/**`, `**/testing/**`, `**/__tests__/**`, `**/spec/**`, `**/specs/**`, + `**/t/**`, `**/testdata/**`, `**/fixtures/**`, `**/*_test.*`, `**/*_tests.*`, `**/test_*.py`, + `**/*Test.java`, `**/*Tests.java`, `**/*Test.kt`, `**/*Test.groovy`, `**/*Spec.groovy`, + `**/*.test.*`, `**/*.spec.*`, `**/*_spec.rb`, `**/*Tests/**`, `**/*.Tests/**`, `**/*.Test/**`, + `**/*.Tests.*/**`, `**/benches/**`, `**/benchmark/**`, `**/benchmarks/**`, `**/vendor/**`, + `**/third_party/**`, `**/thirdparty/**`, `**/third-party/**`, `**/node_modules/**`, `**/deps/**`, + `**/dist/**`, `**/build/**`, `**/generated/**`, `**/*.min.js`, `**/*.d.ts`, `**/*.generated.*`, + `**/*.demo.*`, `**/*.story.*`, `**/*.stories.*`, `**/demos/**`, `**/*-tests/**`, `**/*.pb.go`, + `**/*_string.go`, `**/*.Designer.cs`, `**/*.designer.cs`. + The per-repository extras in the corpus table exclude a duplicate source tree (Guava's `android/` + backport), vendored libraries (three.js `libs/`), generated validators (webpack), and sample apps. + Files `bca` recognises as generated are skipped on top of this. +- Values are each `function`-kind space's own `metrics.halstead.effort`, walked through nested spaces. + That is the space kind `halstead.effort`'s `Function` scope admits in `src/metric_catalog.rs`. + Percentiles use linear interpolation between closest ranks. + +## Result + +| Language | Functions | p50 | p90 | p97.5 | % > 50,000 | p97.5 at v2.3.0 | % > 50,000 at v2.3.0 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Bash | 1,803 | 803 | 8,699 | 44,195 | 2.1 | 44,195 | 2.1 | +| C | 6,361 | 6,049 | 63,982 | 190,936 | 12.5 | 173,763 | 11.3 | +| C# | 6,214 | 375 | 6,914 | 43,715 | 2.3 | 43,957 | 2.3 | +| C++ | 2,491 | 708 | 13,327 | 53,558 | 2.8 | 49,790 | 2.4 | +| Elixir | 4,535 | 460 | 4,398 | 15,121 | 0.4 | 15,121 | 0.4 | +| Go | 9,454 | 1,310 | 21,802 | 89,276 | 4.7 | 89,276 | 4.7 | +| Groovy | 12,529 | 221 | 5,275 | 25,353 | 1.4 | 23,038 | 1.3 | +| Java | 18,198 | 523 | 5,166 | 19,870 | 0.8 | 16,936 | 0.6 | +| JavaScript | 31,030 | 1,191 | 32,475 | 174,060 | 7.2 | 174,108 | 7.3 | +| Kotlin | 6,682 | 428 | 4,945 | 18,334 | 0.6 | 17,413 | 0.6 | +| Lua | 4,562 | 2,673 | 27,416 | 81,544 | 5.1 | 81,544 | 5.1 | +| Objective-C | 1,209 | 1,351 | 16,764 | 76,893 | 4.4 | 75,726 | 4.1 | +| Perl | 1,760 | 640 | 12,537 | 41,205 | 2.0 | 43,862 | 2.2 | +| PHP | 20,296 | 625 | 6,857 | 26,333 | 1.2 | 26,333 | 1.2 | +| Python | 10,264 | 1,030 | 10,904 | 41,983 | 2.0 | 28,535 | 1.2 | +| Ruby | 11,837 | 453 | 2,774 | 4,974 | 0.0 | 4,974 | 0.0 | +| Rust | 8,778 | 998 | 10,472 | 33,894 | 1.4 | 33,309 | 1.3 | +| Tcl | 10,907 | 804 | 13,412 | 45,543 | 2.3 | 45,537 | 2.3 | +| TSX | 6,456 | 774 | 26,726 | 134,616 | 6.0 | 131,020 | 6.0 | +| TypeScript | 6,945 | 830 | 14,724 | 60,339 | 3.1 | 60,126 | 3.0 | + +At the shipped limit of 50,000, the median language flags 2.20% of its functions, inside the 1–3% +band the sizing rule asks for, so the default stands. Candidate limits for comparison: 40,000 gives +2.78% and 60,000 gives 1.76%. The C-family re-measure moved none of these three figures, +because C, C++, and Objective-C rank above the middle pair at every limit before and after it. +The `v2.3.0` binary gives 2.23% on the same corpus. The vocabulary +changes therefore barely moved the default's footing. Their largest per-language effect is Python +(p97.5 28,535 → 41,983), followed by Java (+17%) and Groovy (+10%). Bash, Elixir, Go, Lua, PHP, and +Ruby did not move at all, and Tcl moved by under 0.1%. + +Ruby's low figures are a property of these two repositories, which hold their own methods short +with [RuboCop](https://docs.rubocop.org/rubocop/cops_metrics.html)'s method-length and ABC cops. +They are not a measurement gap. + +## Corpus + +| Language | Repository | Commit | Files | Functions | Extra excludes | +| --- | --- | --- | --- | --- | --- | +| Bash | [nvm-sh/nvm](https://github.com/nvm-sh/nvm) | `b1b3a0038c0666c7436cb5d383608b4b72c8300b` | 4 | 154 | | +| Bash | [ohmybash/oh-my-bash](https://github.com/ohmybash/oh-my-bash) | `abf846186ab0a8a41ec5888e827ece6277dfe446` | 218 | 765 | | +| Bash | [Bash-it/bash-it](https://github.com/Bash-it/bash-it) | `81929690975f9a7ea3c50b87265f2959327bb0a5` | 334 | 884 | | +| C | [curl/curl](https://github.com/curl/curl) | `b8f71d6e1199701352c498467c17847237d1c43e` | 372 | 4,718 | | +| C | [libuv/libuv](https://github.com/libuv/libuv) | `4080d7d965fdf291f37ec72677fc5a2ace8fa361` | 120 | 1,643 | | +| C++ | [fmtlib/fmt](https://github.com/fmtlib/fmt) | `35c58f084e7cd79b51997439cff68346f4c724c0` | 20 | 1,426 | | +| C++ | [google/leveldb](https://github.com/google/leveldb) | `bb74ef739973a70ca9f0d90788a1da89b788c0ad` | 98 | 1,065 | | +| C# | [JamesNK/Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) | `52fa3aef1f2cadcd3a3f874251eddc98d3efbbaa` | 243 | 4,129 | | +| C# | [AutoMapper/AutoMapper](https://github.com/AutoMapper/AutoMapper) | `6e8697bc44f02fb54ef6a556a7d134e63485299e` | 92 | 2,085 | | +| Elixir | [phoenixframework/phoenix](https://github.com/phoenixframework/phoenix) | `9f2c1409ca8bb9d55ca5b9cb7b4244ff645fc2d3` | 85 | 1,688 | | +| Elixir | [elixir-ecto/ecto](https://github.com/elixir-ecto/ecto) | `94d69279c517347ff0962b138f4ccd0556486ae2` | 60 | 2,847 | | +| Go | [caddyserver/caddy](https://github.com/caddyserver/caddy) | `d4165a4eb1fdb129e8ce8390099021a00239ecf7` | 225 | 2,603 | | +| Go | [gohugoio/hugo](https://github.com/gohugoio/hugo) | `0732eadd5aead94d4bd4674d87e9bfc79aaf10a7` | 511 | 6,851 | | +| Groovy | [nextflow-io/nextflow](https://github.com/nextflow-io/nextflow) | `ca258a4e9bce2763001dc67770e2bd957ccf1034` | 967 | 9,815 | | +| Groovy | [apache/groovy](https://github.com/apache/groovy) | `000449404bb0fa255c8bd55cbb35e1c3c7edd379` | 399 | 2,714 | | +| Java | [apache/commons-lang](https://github.com/apache/commons-lang) | `dd2c9e85784b288c7507c3736818040b3b457842` | 264 | 4,550 | | +| Java | [google/guava](https://github.com/google/guava) | `6b3a8d7498a7fde122967ba37a023ba3dcb47792` | 666 | 13,648 | `**/android/**` | +| JavaScript | [mrdoob/three.js](https://github.com/mrdoob/three.js) | `11738a9d55e502fb484cefffcea3a66dc44e9fe9` | 1,403 | 14,891 | `**/libs/**` | +| JavaScript | [webpack/webpack](https://github.com/webpack/webpack) | `5ba5dba8b82a0ebd339a5c2be56166400de3460d` | 1,297 | 16,139 | `**/*.check.js` | +| Kotlin | [Kotlin/kotlinx.coroutines](https://github.com/Kotlin/kotlinx.coroutines) | `bd2e9a1b90400fb7b2fa4f8731e7d8148799ab73` | 342 | 3,029 | | +| Kotlin | [square/okhttp](https://github.com/square/okhttp) | `42e5888d45623662c138ed27e80ea28bbe0aa5cc` | 367 | 3,653 | | +| Lua | [Kong/kong](https://github.com/Kong/kong) | `2d2e67ea7a90cc9d81c53177e9fdbe61bd142236` | 609 | 3,722 | | +| Lua | [luarocks/luarocks](https://github.com/luarocks/luarocks) | `a422d213b38c488472ec3bec70f372bb63784cdb` | 96 | 840 | | +| Objective-C | [SDWebImage/SDWebImage](https://github.com/SDWebImage/SDWebImage) | `c3ad5e1a9bf55c9b76d4c362430b5fcded96c502` | 73 | 889 | `**/Examples/**` | +| Objective-C | [CocoaLumberjack/CocoaLumberjack](https://github.com/CocoaLumberjack/CocoaLumberjack) | `b75d3c3198a2930b4491b04fd47cb504b359c750` | 22 | 320 | `**/Demos/**` | +| Perl | [mojolicious/mojo](https://github.com/mojolicious/mojo) | `da40b9c04e31a530b05ce285fca322e135b3b780` | 124 | 1,569 | | +| Perl | [beyondgrep/ack3](https://github.com/beyondgrep/ack3) | `11d698f9407c6bd5dfcff1c264bf0aa60abacc5d` | 27 | 191 | | +| PHP | [laravel/framework](https://github.com/laravel/framework) | `73a25a4ad3d1da83c3c62a100b8902da380015ef` | 1,820 | 17,242 | | +| PHP | [composer/composer](https://github.com/composer/composer) | `43f74517021a59ed4054a801c1472ef66355c49a` | 315 | 3,054 | | +| Python | [django/django](https://github.com/django/django) | `dab0a5c47f159eb58472110dcbc10bd4a9b01adb` | 763 | 9,324 | | +| Python | [Textualize/rich](https://github.com/Textualize/rich) | `9d8f9a372cc5916fd4781fec207ced7ddac2f08f` | 143 | 940 | | +| Ruby | [rubocop/rubocop](https://github.com/rubocop/rubocop) | `6fa180158e70aaa390fd9f8990b273c5bd38f9b8` | 958 | 10,548 | | +| Ruby | [jekyll/jekyll](https://github.com/jekyll/jekyll) | `541d8b2ee75c8907de4744d4b87a7a6f1f997cae` | 97 | 1,289 | | +| Rust | [BurntSushi/ripgrep](https://github.com/BurntSushi/ripgrep) | `3fce3b5bb0236da2df6d99672afb8a719642eca7` | 91 | 3,219 | | +| Rust | [tokio-rs/tokio](https://github.com/tokio-rs/tokio) | `09a57c27ea64bb546fe16822b9d2e65c599f0fb4` | 488 | 5,559 | | +| Tcl | [tcltk/tcllib](https://github.com/tcltk/tcllib) | `48b8b5dbdf23ced85530a8a2fe8aab1f4bf09f34` | 770 | 7,510 | | +| Tcl | [tcltk/tklib](https://github.com/tcltk/tklib) | `b9f737163cd02f5b243df6c3342fa26e710ab898` | 445 | 3,397 | | +| TypeScript | [vuejs/core](https://github.com/vuejs/core) | `4ab865a848a1da3d10fb674f857e5fff13094644` | 267 | 2,650 | | +| TypeScript | [nestjs/nest](https://github.com/nestjs/nest) | `2731c18054fb348779311d81e4e20949e313be78` | 1,071 | 4,295 | `**/sample/**` | +| TSX | [excalidraw/excalidraw](https://github.com/excalidraw/excalidraw) | `e65eca999e071503efb893d3379a29fb96ff37c4` | 233 | 2,575 | | +| TSX | [mantinedev/mantine](https://github.com/mantinedev/mantine) | `f7ab1ef52579366e2310fc12b1a12e88a821053a` | 759 | 3,881 | | diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index e0c9d6a82..1271d70f9 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -53,16 +53,17 @@ pub mod nested; /// Node-kind filters handed to [`Ast::count`] and [`Ast::find`]. /// -/// `"function"` and `"string"` earn their places twice over: they are the -/// two filters that reach a `Checker` predicate taking `code`, and both -/// consult the ancestor chain `find` and `count` thread (#1381) — the -/// Tcl-family `"string"` on every braced word — so they are the filters -/// a deeply-nested or wide input exercises hardest. +/// `"call"`, `"function"` and `"string"` earn their places twice over: +/// they are the filters that reach a `Checker` predicate taking `code`, +/// and each consults the ancestor chain `find` and `count` thread +/// (#1381, #1580) — the Tcl-family `"string"` on every braced word, the +/// Perl `"call"` on every bareword — so they are the filters a +/// deeply-nested or wide input exercises hardest. /// /// **The order is load-bearing, and `"all"` must stay last.** /// `Filter::any` returns on its first matching predicate, and `"all"` is /// `|_| true`, so listing it first makes every other entry unreachable — -/// `is_call`, `is_comment`, `is_error`, `is_string_with_code` and +/// `is_call_with_code`, `is_comment`, `is_error`, `is_string_with_code` and /// `is_func_with_code` are then never called on any node, in any target. /// It was first here until a review caught it, which had quietly reduced /// `count` and `find` to a bare walk and left the `"function"` predicate diff --git a/src/macros/mod.rs b/src/macros/mod.rs index c2c3efb50..603417d3e 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -48,7 +48,7 @@ macro_rules! implement_metric_trait { ); // Internal helper: shared no-op body for traits whose `compute` // signature is `<'a>(&Node<'a>, &'a [u8], Ancestors<'a, '_>, - // &mut Stats)` (Abc, Cyclomatic). Public arms below delegate here + // &mut Stats)` (Abc, Cyclomatic, Exit). Public arms below delegate here // so the body is written once. `Npa` and `Npm` share the signature // but need `HAS_MEMBERS = false` as well, so they route through // `@code_and_chain_taking_memberless` instead — reaching for this @@ -66,14 +66,8 @@ macro_rules! implement_metric_trait { } )+ ); - // `Exit` is the one metric whose `compute` still takes no ancestor - // chain: no language's exit rule asks what encloses the node. (Exit, $($code:ident),+) => ( - $( - impl Exit for $code { - fn compute<'a>(_node: &Node<'a>, _code: &'a [u8], _stats: &mut Stats) {} - } - )+ + implement_metric_trait!(@code_and_chain_taking Exit, $($code),+); ); (Cyclomatic, $($code:ident),+) => ( implement_metric_trait!(@code_and_chain_taking Cyclomatic, $($code),+); diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index f16f646c6..333b30598 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -501,7 +501,7 @@ pub(super) fn peel<'a>(node: &Node<'a>, step: PeelStep<'a>) -> (Node<'a>, bool) pub(super) fn count_boolean_slot<'a>( slot: &Node<'a>, step: PeelStep<'a>, - scores_itself: fn(&Node<'a>) -> bool, + scores_itself: impl Fn(&Node<'a>) -> bool, conditions: &mut f64, ) { if !scores_itself(&peel(slot, step).0) { @@ -527,16 +527,18 @@ pub(super) fn count_negated_operand<'a>( } } -// A language's scorer for one operand: its `count_boolean_slot` or -// `count_negated_operand` wrapper. -pub(super) type OperandScorer = fn(&Node, &mut f64); - // Fitzpatrick Rule 9 for a chain whose grammar names its two operands -// `left` and `right`: each is scored by `score`. Read by field rather +// `left` and `right`: each is scored by `score`, the language's +// `count_boolean_slot` or `count_negated_operand` wrapper — a closure +// where it needs the source bytes (Elixir's, #1574). Read by field rather // than as every operand child, so a comment beside the operator — a // named child too — does not pay. The caller owns the gate deciding // that `chain` is a chain at all. -pub(super) fn count_field_operands(chain: &Node, score: OperandScorer, conditions: &mut f64) { +pub(super) fn count_field_operands( + chain: &Node, + score: impl Fn(&Node, &mut f64), + conditions: &mut f64, +) { for field in ["left", "right"] { if let Some(operand) = chain.child_by_field_name(field) { score(&operand, conditions); @@ -547,7 +549,11 @@ pub(super) fn count_field_operands(chain: &Node, score: OperandScorer, condition // Each operand child of `list` — an argument list, a `return` value // list, a chain whose grammar names no operand field — scored by // `score`. `is_operand` keeps punctuation and comments out. -pub(super) fn count_each_operand(list: &Node, score: OperandScorer, conditions: &mut f64) { +pub(super) fn count_each_operand( + list: &Node, + score: impl Fn(&Node, &mut f64), + conditions: &mut f64, +) { for operand in list.children().filter(is_operand) { score(&operand, conditions); } @@ -11710,9 +11716,10 @@ end // clause's flat one beside its `>`, `rescue_in` a sole pattern that // scores itself (the `in` arm), `rescue_alias` a sole pattern the // clause slot pays for, `catch_two` a multi-pattern clause, `recv` a - // `receive`'s `after` timeout clause, and `with_else` the `else` of - // a `with`, whose clauses pay rather than a Rule 5 `else`. Its `<-` - // clause pays one more since #1535, in both metrics. + // `receive`'s `after` timeout clause — free since #1581, as the + // default arm, so `recv` scores what `case2` does — and `with_else` + // the `else` of a `with`, whose clauses pay rather than a Rule 5 + // `else`. Its `<-` clause pays one more since #1535, in both metrics. #[cfg(feature = "elixir")] #[test] fn elixir_clause_constructs_pay_per_clause() { @@ -11785,7 +11792,7 @@ end ("case_default", 2, 3), ("case_guard", 2, 3), ("with_else", 2, 3), - ("recv", 3, 4), + ("recv", 2, 3), ("rescue_in", 1, 2), ("rescue_alias", 1, 2), ("catch_two", 1, 2), diff --git a/src/metrics/abc/cpp.rs b/src/metrics/abc/cpp.rs index 6753adf38..5093860e8 100644 --- a/src/metrics/abc/cpp.rs +++ b/src/metrics/abc/cpp.rs @@ -11,7 +11,7 @@ use super::{ Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, - count_negated_operand, is_operand, wrapped_operand, + count_negated_operand, is_operand, last_operand, wrapped_operand, }; use crate::metrics::cyclomatic::cpp_operator_is_applied; use crate::*; @@ -51,7 +51,11 @@ fn cpp_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { .is_some_and(|op| accept.contains(&op.kind())) }; let value = match node.kind() { - "parenthesized_expression" => wrapped_operand(node), + "parenthesized_expression" | "co_await_expression" => wrapped_operand(node), + // A GNU statement expression yields only its final expression statement. + "compound_statement" => last_operand(node) + .filter(|statement| statement.kind() == "expression_statement") + .and_then(|statement| wrapped_operand(&statement)), "condition_clause" | "cast_expression" | "declaration" => node.child_by_field_name("value"), "initializer_list" => sole_operand(node), "comma_expression" => node.child_by_field_name("right"), diff --git a/src/metrics/abc/csharp.rs b/src/metrics/abc/csharp.rs index 5ceee2fbd..99a8812fe 100644 --- a/src/metrics/abc/csharp.rs +++ b/src/metrics/abc/csharp.rs @@ -78,8 +78,9 @@ fn csharp_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { use Csharp::*; match node.kind_id().into() { - // `(expr)` — the inner expression follows the `(` token. + // Parentheses, checked/unchecked and await preserve the value's condition. csharp_paren_expr_kinds!() => wrapped_operand(node).map(|o| (o, false)), + CheckedExpression | AwaitExpression => wrapped_operand(node).map(|o| (o, false)), // `!expr` — the operand follows the operator token. Seven other // prefix operators (`++ -- + - ~ & ^`) share this kind, as does // the `*` of a pointer indirection the grammar aliases onto it; diff --git a/src/metrics/abc/elixir.rs b/src/metrics/abc/elixir.rs index 300eefa45..94a72790e 100644 --- a/src/metrics/abc/elixir.rs +++ b/src/metrics/abc/elixir.rs @@ -9,11 +9,17 @@ clippy::cast_sign_loss )] -use super::{Abc, Stats, count_boolean_slot, count_field_operands, last_operand, wrapped_operand}; -use crate::lang_helpers::elixir::{elixir_applying_operator, elixir_call_keyword}; +use super::{ + Abc, Stats, count_boolean_slot, count_each_operand, count_field_operands, last_operand, + wrapped_operand, +}; +use crate::lang_helpers::elixir::{ + elixir_applying_operator, elixir_applying_remote_call, elixir_call_keyword, + elixir_kernel_short_circuit_name, +}; use crate::metrics::elixir_decisions::{ - elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_sole_unguarded_pattern, - elixir_when_alternative, elixir_when_is_guard, + Section, elixir_arrow_is_with_clause, elixir_call_has_section, elixir_clause_is_decision, + elixir_sole_unguarded_pattern, elixir_when_alternative, elixir_when_is_guard, }; use crate::*; @@ -71,33 +77,37 @@ fn elixir_binary_operator_is(node: &Node, accept: impl Fn(Elixir) -> bool) -> bo // arms) and for an `&&` / `||` / `and` / `or` chain, whose walker scores // each operand through `elixir_count_condition` — so every operand // either pays there or is one of these, and a chain always carries at -// least one condition of its own. +// least one condition of its own. A short-circuit `Kernel` remote call +// (`Kernel.||(a, b)`, `Kernel.or(a, b)`) is that chain spelled as a +// call, and its operands pay through `elixir_count_remote_chain` in the +// same way (#1574, #1582). // // The operator list must agree with the token arms in `compute`: an // operator listed here and counted nowhere would leave its slot at zero, // and one counted there but missing here would pay twice. -fn elixir_condition_scores_itself(expr: &Node) -> bool { +fn elixir_condition_scores_itself(expr: &Node, code: &[u8]) -> bool { use Elixir as E; - elixir_binary_operator_is(expr, |op| { - matches!( - op, - E::EQEQ - | E::EQEQEQ - | E::BANGEQ - | E::BANGEQEQ - | E::LTEQ - | E::GTEQ - | E::LT - | E::GT - | E::In - | E::Notin - | E::AMPAMP - | E::PIPEPIPE - | E::And - | E::Or - ) - }) + elixir_kernel_short_circuit_name(expr, code).is_some() + || elixir_binary_operator_is(expr, |op| { + matches!( + op, + E::EQEQ + | E::EQEQEQ + | E::BANGEQ + | E::BANGEQEQ + | E::LTEQ + | E::GTEQ + | E::LT + | E::GT + | E::In + | E::Notin + | E::AMPAMP + | E::PIPEPIPE + | E::And + | E::Or + ) + }) } // Scores one boolean slot — an `if` / `unless` predicate, a `cond` @@ -115,11 +125,11 @@ fn elixir_condition_scores_itself(expr: &Node) -> bool { // whether the decision is already paid for, rather than whether the // occupant is a known terminal, also scores what the list never held — // `a && x + 1`, `a && @flag`, `a && -x` each scored one below `a && b`. -fn elixir_count_condition(condition: &Node, conditions: &mut f64) { +fn elixir_count_condition(condition: &Node, code: &[u8], conditions: &mut f64) { count_boolean_slot( condition, elixir_wrapper_operand, - elixir_condition_scores_itself, + |expr| elixir_condition_scores_itself(expr, code), conditions, ); } @@ -139,9 +149,9 @@ fn elixir_count_condition(condition: &Node, conditions: &mut f64) { // `binary_operator` has no `right` child, which the grammar declares // required — so the `if let`'s else is unreachable at the pin rather // than untested. -fn elixir_count_guard(when_operator: &Node, conditions: &mut f64) { +fn elixir_count_guard(when_operator: &Node, code: &[u8], conditions: &mut f64) { if let Some(alternative) = elixir_when_alternative(when_operator) { - elixir_count_condition(&alternative, conditions); + elixir_count_condition(&alternative, code, conditions); } } @@ -163,31 +173,55 @@ fn elixir_count_guard(when_operator: &Node, conditions: &mut f64) { // clause (`x when g ->`, `kind, reason ->`) pays one for its pattern // match, and a guard pays its own slot through the `when` arm, as the // two `Cyclomatic` decisions it carries. -fn elixir_count_clause(clause: &Node, conditions: &mut f64) { +fn elixir_count_clause(clause: &Node, code: &[u8], conditions: &mut f64) { match elixir_sole_unguarded_pattern(clause) { - Some(pattern) => elixir_count_condition(&pattern, conditions), + Some(pattern) => elixir_count_condition(&pattern, code, conditions), None => *conditions += 1., } } -// Whether an `if` / `unless` Call carries an `else` branch, in either +// Whether an `if` / `unless` Call carries an `else` branch, in any // spelling: the block form's `else_block` inside the `do_block` -// (`if b do … else … end`), or the keyword form's `else:` pair -// (`if b, do: …, else: …`). The keyword token's text carries its colon -// and the whitespace after it, so it is trimmed before comparing. +// (`if b do … else … end`), or the keyword form's `else:` pair, bare +// (`if b, do: …, else: …`) or bracketed (`if(b, [do: …, else: …])`). fn elixir_has_else(call: &Node, code: &[u8]) -> bool { - use Elixir as E; + elixir_call_has_section(call, code, Section::Else) +} - call.children().any(|child| match child.kind_id().into() { - E::DoBlock => child.children().any(|c| c.kind_id() == E::ElseBlock as u16), - _ if child.kind() == "arguments" => child - .children() - .filter(|c| c.kind_id() == E::Keywords as u16) - .flat_map(|keywords| keywords.children()) - .filter_map(|pair| pair.child_by_field_name("key")) - .any(|key| key.utf8_text(code).is_some_and(|k| k.trim_end() == "else:")), - _ => false, - }) +// The operands of a short-circuit `Kernel` remote call +// (`elixir_applying_remote_call`), each a boolean slot as the `left` and +// `right` of `a || b` are, so `Kernel.||(a, b)` scores what `a || b` +// does (#1574): the call's arguments, and under a pipe +// (`a |> Kernel.||(b)`) the piped value, which the pipe passes as the +// first argument. `up` yields the ancestors above the call. +// +// Which side of the pipe the call sits on is not checked: in +// `Kernel.||(a, b) |> f()` the piped value is the call itself, which +// `elixir_condition_scores_itself` already says pays nothing. +// +// `op` is the operator's name, a token or the `identifier` of +// `Kernel.and` / `Kernel.or` (#1582); any other node scores nothing. +fn elixir_count_remote_chain<'a>( + op: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, + conditions: &mut f64, +) { + let Some(remote) = elixir_applying_remote_call(op, code, ancestors) else { + return; + }; + let score = |operand: &Node, conditions: &mut f64| { + elixir_count_condition(operand, code, conditions); + }; + if let Some(piped) = remote + .ancestors + .parent(&remote.call) + .filter(|pipe| elixir_binary_operator_is(pipe, |op| op == Elixir::PIPEGT)) + .and_then(|pipe| pipe.child_by_field_name("left")) + { + score(&piped, conditions); + } + count_each_operand(&remote.arguments, score, conditions); } // What an Elixir `Call` contributes. The classification is by keyword @@ -216,6 +250,12 @@ fn elixir_count_call(node: &Node, code: &[u8], stats: &mut Stats) { if !is_definition_or_directive { stats.branches += 1.; } + if keyword == Some("receive") + && let Some(timeout) = + crate::metrics::elixir_decisions::elixir_receive_timeout_clause(node, code) + { + elixir_count_clause(&timeout, code, &mut stats.conditions); + } // The predicate is the Call's first argument in both the block // (`if p do … end`) and keyword (`if p, do: …`) forms, and is a // slot (#1527). `arguments` carries five kind aliases at this @@ -226,7 +266,7 @@ fn elixir_count_call(node: &Node, code: &[u8], stats: &mut Stats) { .find(|c| c.kind() == "arguments") .and_then(|args| wrapped_operand(&args)) { - elixir_count_condition(&predicate, &mut stats.conditions); + elixir_count_condition(&predicate, code, &mut stats.conditions); } // Fitzpatrick Rule 5 counts the `else`, as Ruby's, Java's, // Python's and Go's `if … else` each pay it (#1531). Only an @@ -367,7 +407,7 @@ impl Abc for ElixirCode { // and `Cyclomatic` impls share. E::When if elixir_when_is_guard(node, code, ancestors) => { if let Some(operator) = ancestors.parent(node) { - elixir_count_guard(&operator, &mut stats.conditions); + elixir_count_guard(&operator, code, &mut stats.conditions); } } // `in` / `not in` are Elixir's membership and type tests @@ -445,14 +485,28 @@ impl Abc for ElixirCode { // token's parent can be an `ERROR` node, which has no // operands to score. `a && b || c` is a left-nested chain of // `binary_operator`s, so an operand that is itself a chain is - // paid by its own operator's visit. + // paid by its own operator's visit. A short-circuit `Kernel` + // remote call applies the operator too, as `Cyclomatic` + // counts it through the same predicate (§7, #1574). E::AMPAMP | E::PIPEPIPE | E::And | E::Or => { if let Some(chain) = elixir_applying_operator(node, ancestors) { - count_field_operands(&chain, elixir_count_condition, &mut stats.conditions); + count_field_operands( + &chain, + |operand, conditions| elixir_count_condition(operand, code, conditions), + &mut stats.conditions, + ); + } else { + elixir_count_remote_chain(node, code, ancestors, &mut stats.conditions); } } + // `Kernel.and(a, b)` / `Kernel.or(a, b)` name the operator + // with an `identifier`, so the token arm above never sees + // them; they are the same chain spelled as a call (#1582). + E::Identifier => { + elixir_count_remote_chain(node, code, ancestors, &mut stats.conditions); + } E::StabClause if elixir_clause_is_decision(node, code, ancestors) => { - elixir_count_clause(node, &mut stats.conditions); + elixir_count_clause(node, code, &mut stats.conditions); } // A `with`'s `pattern <- expr` clause is a pattern-match // condition (Rule 9), the decision `Cyclomatic` counts diff --git a/src/metrics/abc/js_family.rs b/src/metrics/abc/js_family.rs index ec0aba7bd..cc4de47e9 100644 --- a/src/metrics/abc/js_family.rs +++ b/src/metrics/abc/js_family.rs @@ -21,7 +21,7 @@ use crate::*; // (grammar-dispatch §1: TypeScript alone emits two ids for // `parenthesized_expression` and for `unary_expression`). // -// One step of the value peel (see `PeelStep`). A parenthesis, a TS type +// One step of the value peel (see `PeelStep`). A parenthesis, an await, a TS type // cast (`x as T`, `x satisfies T`, `x`) and a non-null assertion // (`x!`) evaluate to their operand; a comma sequence to its last // operand; an assignment to its `right` value, so `if (y = x > 1)` tests @@ -34,6 +34,7 @@ use crate::*; fn js_family_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { match node.kind() { "parenthesized_expression" + | "await_expression" | "as_expression" | "satisfies_expression" | "non_null_expression" => wrapped_operand(node).map(|o| (o, false)), diff --git a/src/metrics/abc/kotlin.rs b/src/metrics/abc/kotlin.rs index 04154e4f7..4059e4855 100644 --- a/src/metrics/abc/kotlin.rs +++ b/src/metrics/abc/kotlin.rs @@ -94,6 +94,12 @@ fn kotlin_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { match node.kind_id().into() { // `(expr)` — the inner expression follows the `(` token. ParenthesizedExpression => wrapped_operand(node).map(|o| (o, false)), + AnnotatedExpression | LabeledExpression => node + .children() + .find(|child| { + super::is_operand(child) && !matches!(child.kind(), "annotation" | "label") + }) + .map(|o| (o, false)), UnaryExpression => { let operand = node.child_by_field_name("argument")?; match node.child_by_field_name("operator")?.kind_id().into() { diff --git a/src/metrics/abc/perl.rs b/src/metrics/abc/perl.rs index 0589a4c4f..84ab0db8c 100644 --- a/src/metrics/abc/perl.rs +++ b/src/metrics/abc/perl.rs @@ -14,7 +14,7 @@ use super::{ wrapped_operand, }; use crate::lang_helpers::perl::{ - perl_and_is_operator, perl_dash_signs_keyword_key, perl_is_dash_key_word, perl_not_is_key, + perl_and_is_operator, perl_bareword_is_call, perl_dash_signs_keyword_key, perl_not_is_key, }; use crate::*; @@ -135,7 +135,13 @@ fn perl_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { P::And if !perl_and_is_operator(&op, node) => { perl_last_operand(node).map(|o| (o, false)) } - P::DASH if perl_dash_signs_keyword_key(&op, node) => { + // The `-`'s parent is all the chain the test reads. + P::DASH + if perl_dash_signs_keyword_key( + &op, + Ancestors::known(std::slice::from_ref(node)), + ) => + { wrapped_operand(node).map(|o| (o, false)) } P::BANG | P::Not => wrapped_operand(node).map(|o| (o, true)), @@ -280,11 +286,12 @@ fn perl_count_slot(slot: Option, conditions: &mut f64) { // not scanned child by child. Without them every spine node climbed to // the top and every chain scanned its statement list, both quadratic. // The `ERROR` is the one wide parent left, holding every chain of an -// unclosed list, so the operator before `child` is found by index rather -// than by a scan from the `ERROR`'s start. +// unclosed list. Recovery builds it as one flat array, which a seek by +// byte scans from its start, so the operator before `child` is read off +// the children the walk recorded (#1584). fn perl_misparse_absorbs_leading_operand<'a>( node: &Node<'a>, - mut ancestors: impl Iterator>, + ancestors: Ancestors<'a, '_>, ) -> bool { use Perl as P; @@ -293,18 +300,18 @@ fn perl_misparse_absorbs_leading_operand<'a>( { return false; } - let mut child = *node; - let spine_parent = ancestors.find(|parent| { + let (mut child, mut child_ancestors) = (*node, ancestors); + let spine_parent = ancestors.iter(node).find(|(parent, above)| { let on_spine = perl_leading_slot(parent).is_some_and(|lead| lead.id() == child.id()); if on_spine { - child = *parent; + (child, child_ancestors) = (*parent, *above); } !on_spine }); - spine_parent.is_some_and(|parent| { + spine_parent.is_some_and(|(parent, _)| { (parent.kind_id() == P::BinaryExpression as u16 || parent.is_error()) - && parent - .children_before(&child) + && child_ancestors + .siblings_before(&child) .find(|sibling| !sibling.as_tree_sitter().is_extra()) .is_some_and(|operator| { matches!( @@ -566,22 +573,12 @@ impl Abc for PerlCode { // when this node is the outermost dispatch site. When the // bareword sits inside one of the wrappers above, the // outer node has already been counted and this child - // would double the branch tally. Nor is the word of a - // `-bareword` key a call (FIXME(#1545 upstream)): the - // grammar splits `$h{-foo}` into a file test on a bareword - // `oo`, and the key is a string. - P::CallExpressionWithBareword - if !ancestors.parent(node).is_some_and(|p| { - matches!( - p.kind_id().into(), - P::CallExpressionWithSpacedArgs - | P::CallExpressionWithSub - | P::CallExpressionWithArgsWithBrackets - | P::CallExpressionWithVariable - | P::CallExpressionRecursive - ) - }) && !perl_is_dash_key_word(node, ancestors) => - { + // would double the branch tally. Nor is an auto-quoted + // key a call (FIXME(#1571 upstream)): the grammar reads + // `(name => 1)`, `(-name => 1)` and `$h{-name}` as calls of + // `name`, and `$h{-foo}` as a file test on a bareword `oo`, + // where Perl holds a string (#1545, #1572). + P::CallExpressionWithBareword if perl_bareword_is_call(node, ancestors) => { stats.branches += 1.; } // Numeric, string, and pattern-match comparison operators @@ -653,8 +650,7 @@ impl Abc for PerlCode { // condition (issue #403). Covers `&&`, `||`, `//`, // `and`, `or` — not `xor`, which is a value (#1536). P::AMPAMP | P::PIPEPIPE | P::SLASHSLASH | P::And | P::Or => { - let mut above = ancestors.iter(node).map(|(ancestor, _)| ancestor); - if let Some(chain) = above.next() { + if let Some((chain, above)) = ancestors.iter(node).next() { let absorbed = perl_misparse_absorbs_leading_operand(&chain, above); perl_count_chain_operands(&chain, absorbed, &mut stats.conditions); } @@ -695,8 +691,7 @@ impl Abc for PerlCode { // and adds the three operand slots (issue #1102). P::TernaryExpression => { stats.conditions += 1.; - let above = ancestors.iter(node).map(|(ancestor, _)| ancestor); - let absorbed = perl_misparse_absorbs_leading_operand(node, above); + let absorbed = perl_misparse_absorbs_leading_operand(node, ancestors); perl_walk_ternary(node, absorbed, &mut stats.conditions); } // Phase-2B (issue #403): condition slots, each read by the diff --git a/src/metrics/abc/python.rs b/src/metrics/abc/python.rs index 0b0169799..643060162 100644 --- a/src/metrics/abc/python.rs +++ b/src/metrics/abc/python.rs @@ -54,7 +54,7 @@ use crate::*; // stops on it instead. fn python_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { match node.kind_id().into() { - Python::ParenthesizedExpression => wrapped_operand(node), + Python::ParenthesizedExpression | Python::Await => wrapped_operand(node), Python::NamedExpression => node.child_by_field_name("value"), _ => None, } diff --git a/src/metrics/abc/rust.rs b/src/metrics/abc/rust.rs index 4d60b66d0..831cd0a68 100644 --- a/src/metrics/abc/rust.rs +++ b/src/metrics/abc/rust.rs @@ -27,11 +27,33 @@ fn rust_wrapper_operand<'a>(node: &Node<'a>) -> Option<(Node<'a>, bool)> { use Rust::*; match node.kind_id().into() { - ParenthesizedExpression => wrapped_operand(node).map(|o| (o, false)), - Block | Block2 => last_operand(node).map(|o| (o, false)), + AwaitExpression => { + // Only an awaited async block yields its body; a bare one is a future. + let mut operand = wrapped_operand(node)?; + while let Some(inner) = rust_transparent_operand(&operand) { + operand = inner; + } + if operand.kind_id() == AsyncBlock as u16 { + operand = wrapped_operand(&operand)?; + } + Some((operand, false)) + } UnaryExpression if node.child(0).is_some_and(|c| c.kind_id() == BANG as u16) => { wrapped_operand(node).map(|o| (o, true)) } + _ => rust_transparent_operand(node).map(|o| (o, false)), + } +} + +fn rust_transparent_operand<'a>(node: &Node<'a>) -> Option> { + use Rust::*; + + match node.kind_id().into() { + ParenthesizedExpression | UnsafeBlock => wrapped_operand(node), + ConstBlock => node.child_by_field_name("body"), + TypeCastExpression => node.child_by_field_name("value"), + Block | Block2 => last_operand(node), + ExpressionStatement if !node.is_child(SEMI as u16) => wrapped_operand(node), _ => None, } } diff --git a/src/metrics/abc/slot_pays_tests.rs b/src/metrics/abc/slot_pays_tests.rs index 8e3575cf3..39f679926 100644 --- a/src/metrics/abc/slot_pays_tests.rs +++ b/src/metrics/abc/slot_pays_tests.rs @@ -16,12 +16,56 @@ use crate::{LANG, Metric, MetricsOptions}; /// The placeholder a template marks its slot with. Not `@`, which /// Objective-C, PHP and Perl predicates spell. +#[cfg(any( + feature = "c", + feature = "cpp", + feature = "csharp", + feature = "elixir", + feature = "groovy", + feature = "irules", + feature = "java", + feature = "javascript", + feature = "kotlin", + feature = "lua", + feature = "mozcpp", + feature = "mozjs", + feature = "objc", + feature = "perl", + feature = "php", + feature = "python", + feature = "ruby", + feature = "rust", + feature = "tcl", + feature = "typescript", +))] const SLOT: &str = "PRED"; /// For each `(predicate, conditions, decisions)` row, replaces the /// template's `SLOT` with the predicate and asserts the function named `f` /// scores `conditions` ABC conditions and `decisions` cyclomatic /// decisions (`cyclomatic - 1`). +#[cfg(any( + feature = "c", + feature = "cpp", + feature = "csharp", + feature = "elixir", + feature = "groovy", + feature = "irules", + feature = "java", + feature = "javascript", + feature = "kotlin", + feature = "lua", + feature = "mozcpp", + feature = "mozjs", + feature = "objc", + feature = "perl", + feature = "php", + feature = "python", + feature = "ruby", + feature = "rust", + feature = "tcl", + feature = "typescript", +))] #[track_caller] fn assert_rows(lang: LANG, template: &str, rows: &[(&str, u64, u64)]) { assert!(template.contains(SLOT), "template has no predicate slot"); @@ -141,6 +185,32 @@ fn python_elif_is_an_else_and_a_slot() { ); } +#[test] +#[cfg(feature = "python")] +fn python_await_preserves_condition_counts() { + let source = r"class AwaitableComparison: + def __gt__(self, other): + async def result(): + return True + return result() + +async def f(a, b): + if PRED: + pass +"; + assert_rows( + LANG::Python, + source, + &[ + ("await b", 1, 1), + ("await (b > 1)", 1, 1), + ("await ( # comment\n b > 1)", 1, 1), + ("not await (b > 1)", 1, 1), + ("a and await (b > 1)", 2, 2), + ], + ); +} + /// The rows every JS-family grammar shares: the issue's predicates, each /// beside the bare identifier and the comparison it must neither fall /// below nor double. @@ -213,6 +283,28 @@ fn assert_js_family_slots(lang: LANG) { &function.replace(SLOT, "g(PRED);"), &[("b", 0, 0), ("!b", 1, 0), ("!this", 1, 0)], ); + assert_rows( + lang, + "async function f(a, b) { if (PRED) {} }", + &[ + ("await b", 1, 1), + ("await (b > 1)", 1, 1), + ("await /* comment */ (b > 1)", 1, 1), + ("await (a && b)", 2, 2), + ("a && await (b > 1)", 2, 2), + ("!await (b > 1)", 1, 1), + ("await await (b > 1)", 1, 1), + ], + ); + assert_rows( + lang, + "async function f(b) { return PRED; }", + &[ + ("await b", 0, 0), + ("await !b", 1, 0), + ("!await (b > 1)", 1, 0), + ], + ); } #[test] @@ -289,6 +381,13 @@ const C_FAMILY_IF_ROWS: &[(&str, u64, u64)] = &[ ("(int)(b > 1)", 1, 1), ("(int)b", 1, 1), ("a, b > 1", 1, 1), + ("({ b; })", 1, 1), + ("({ b > 1; })", 1, 1), + ("({ a; /* before */ b > 1; /* after */ })", 1, 1), + ("({ b > 1; a; })", 2, 1), + ("({ a > 1 && b > 1; })", 2, 2), + ("a && ({ b > 1; })", 2, 2), + ("!({ b > 1; })", 1, 1), ("!(b > 1)", 1, 1), ("b > 1", 1, 1), ("!b", 1, 1), @@ -338,6 +437,8 @@ fn assert_c_family_slots(lang: LANG) { ("!*p", 1, 0), ("!\"s\"", 1, 0), ("!(b > 1)", 1, 0), + ("({ !b; })", 1, 0), + ("!({ b > 1; })", 1, 0), ], ); } @@ -441,6 +542,31 @@ fn assert_cpp_slots(lang: LANG) { "template bool f(T a) requires PRED { return a && a; }", &with_body, ); + assert_rows( + lang, + r"#include +struct Task { + struct promise_type { + Task get_return_object() { return {}; } + std::suspend_never initial_suspend() { return {}; } + std::suspend_never final_suspend() noexcept { return {}; } + void return_void() {} + void unhandled_exception() {} + }; +}; +struct Awaitable { + bool await_ready() const { return true; } + void await_suspend(std::coroutine_handle<>) const {} + bool await_resume() const { return true; } + Awaitable operator>(int) const { return {}; } +}; +Task f(Awaitable b) { if (PRED) {} co_return; }", + &[ + ("co_await b", 1, 1), + ("co_await (b > 1)", 1, 1), + ("co_await /* comment */ (b > 1)", 1, 1), + ], + ); } #[test] @@ -490,6 +616,16 @@ fn rust_slots_pay_for_any_predicate() { // A block evaluates to its last expression: the comparison // inside pays, the slot does not pay again. ("{ x > 1 }", 1, 1), + ("unsafe { x > 1 }", 1, 1), + ("unsafe { /* comment */ x > 1 }", 1, 1), + ("const { 2 > 1 }", 1, 1), + ("const { true }", 1, 1), + ("b as bool", 1, 1), + ("(x > 1) as bool", 1, 1), + ("(a && b) as bool", 2, 2), + ("unsafe { a && b }", 2, 2), + ("a && unsafe { x > 1 }", 2, 2), + ("unsafe { x > 1; a }", 2, 1), // The match arm is a decision of its own; the slot tests the // value the match yields. ("match x { 1 => true, _ => false }", 2, 2), @@ -527,7 +663,36 @@ fn rust_slots_pay_for_any_predicate() { assert_rows( LANG::Rust, &function.replace(SLOT, "g(PRED);"), - &[("*p", 0, 0), ("!*p", 1, 0), ("!(a & b)", 1, 0)], + &[ + ("*p", 0, 0), + ("!*p", 1, 0), + ("!(a & b)", 1, 0), + ("unsafe { !b }", 1, 0), + ("!unsafe { x > 1 }", 1, 0), + ], + ); + assert_rows( + LANG::Rust, + "async fn f(a: bool, x: i32) { if PRED {} }", + &[ + ("async { a }.await", 1, 1), + ("async { x > 1 }.await", 1, 1), + ("((async { x > 1 })).await", 1, 1), + ("{ async { x > 1 } }.await", 1, 1), + ("unsafe { async { x > 1 } }.await", 1, 1), + ("async move { x > 1 }.await", 1, 1), + ("async { a && x > 1 }.await", 2, 2), + ("!async { x > 1 }.await", 1, 1), + ], + ); + assert_rows( + LANG::Rust, + "async fn f(b: bool) { g(PRED); }", + &[ + ("async { !b }", 0, 0), + ("async { !b }.await", 1, 0), + ("!async { !b }.await", 1, 0), + ], ); } @@ -753,6 +918,62 @@ fn csharp_1475_constructs_pay_their_slot() { ); } +#[test] +#[cfg(feature = "csharp")] +fn csharp_checked_wrappers_preserve_condition_counts() { + for wrapper in ["checked", "unchecked"] { + let function = format!( + "class C {{ bool f(bool a, bool b, int x) {{ if ({wrapper}(PRED)) {{}} return false; }} }}" + ); + assert_rows( + LANG::Csharp, + &function, + &[ + ("b", 1, 1), + ("x > 1", 1, 1), + ("/* comment */ x > 1", 1, 1), + ("a && b", 2, 2), + ("!(x > 1)", 1, 1), + ("checked(unchecked(x > 1))", 1, 1), + ], + ); + assert_rows( + LANG::Csharp, + &format!("class C {{ bool f(bool b, int x) {{ return {wrapper}(PRED); }} }}"), + &[("b", 0, 0), ("!b", 1, 0), ("!(x > 1)", 1, 0)], + ); + } + assert_rows( + LANG::Csharp, + "class C { async Task f(bool a, Task b, Task c) \ + { if (PRED) {} return false; } }", + &[ + ("await b", 1, 1), + ("await (a ? b : c)", 2, 2), + ("await /* comment */ (a ? b : c)", 2, 2), + ], + ); +} + +#[test] +#[cfg(feature = "kotlin")] +fn kotlin_labeled_and_annotated_wrappers_preserve_condition_counts() { + for wrapper in ["pred@ (PRED)", "@Suppress(\"unused\") (PRED)"] { + let template = format!("fun f(a: Boolean, b: Boolean, x: Int) {{ if ({wrapper}) {{}} }}"); + assert_rows( + LANG::Kotlin, + &template, + &[ + ("b", 1, 1), + ("x > 1", 1, 1), + ("/* comment */ x > 1", 1, 1), + ("a && b", 2, 2), + ("!(x > 1)", 1, 1), + ], + ); + } +} + /// PHP is truthy-valued, so every predicate is a decision. `??` is no /// condition token in PHP, so a slot holding one pays for it. #[test] diff --git a/src/metrics/bash_test_connective_tests.rs b/src/metrics/bash_test_connective_tests.rs index fe1e37106..95b7eb0d8 100644 --- a/src/metrics/bash_test_connective_tests.rs +++ b/src/metrics/bash_test_connective_tests.rs @@ -220,3 +220,22 @@ fn bash_comparison_runs_score_like_their_twins_1536() { ); } } + +#[cfg(feature = "bash")] +#[test] +fn bash_unary_test_preserves_the_comparison_sequence() { + for (operator, connective) in [("-a", "&&"), ("-o", "||")] { + for unary in [r#"-n "$b""#, r#"! -z "$b""#, r#"-f "$b""#] { + for comparison in ["=", "==", "!="] { + let single = format!( + r#"if [ {unary} {operator} "$a" {comparison} 1 {operator} "$c" {comparison} 2 ]; then :; fi"# + ); + let twin = format!( + r#"if [ {unary} ] {connective} [ "$a" {comparison} 1 ] {connective} [ "$c" {comparison} 2 ]; then :; fi"# + ); + assert_eq!(measure(&twin), [5, 4, 2], "{twin}"); + assert_eq!(measure(&single), [5, 4, 2], "{single}"); + } + } + } +} diff --git a/src/metrics/cognitive/bash.rs b/src/metrics/cognitive/bash.rs index 751501ed0..be07e9c06 100644 --- a/src/metrics/cognitive/bash.rs +++ b/src/metrics/cognitive/bash.rs @@ -39,10 +39,9 @@ const RUN_COMPARISONS: [u16; 4] = [ ]; /// The comparison runs the walk is inside, innermost last. A run member -/// is a `BinaryExpression3` whose operator is one of -/// [`RUN_COMPARISONS`]; each records its span and the end of the -/// outermost member of its run, inherited when its parent is a member -/// too. +/// is a comparison or an intervening unary test; each records its span +/// and the end of the outermost member of its run, inherited when its +/// parent is a member too. /// /// The walk is pre-order, so a member is visited before everything under /// it, and one whose span does not hold the node being visited is @@ -65,7 +64,7 @@ struct RunMember { } impl ComparisonRuns { - /// The byte a boolean sequence under `node`, a `BinaryExpression3` + /// The byte a boolean sequence under a binary expression or unary test /// whose parent is `parent`, extends to: the end of the run `parent` /// is a member of, else `node`'s own end. Records `node` when it is a /// member itself. @@ -82,7 +81,7 @@ impl ComparisonRuns { (Some(parent), Some(member)) if member.id == parent.id() => member.run_end, _ => end, }; - if node.wraps_any(&RUN_COMPARISONS) { + if node.kind() == "unary_expression" || node.wraps_any(&RUN_COMPARISONS) { self.0.push(RunMember { id: node.id(), start, @@ -140,6 +139,9 @@ impl Cognitive for BashCode { List => { compute_booleans(node, stats, AMPAMP, PIPEPIPE); } + UnaryExpression | UnaryExpression2 | UnaryExpression3 => { + stats.bash_runs.sequence_end(node, ancestors.parent(node)); + } BinaryExpression3 => { let end = stats.bash_runs.sequence_end(node, ancestors.parent(node)); compute_booleans_by_node(node, end, stats, |child| match child.kind_id().into() { diff --git a/src/metrics/cognitive/elixir.rs b/src/metrics/cognitive/elixir.rs index b5facbd8f..a2c89242d 100644 --- a/src/metrics/cognitive/elixir.rs +++ b/src/metrics/cognitive/elixir.rs @@ -12,23 +12,83 @@ )] use super::*; +use crate::lang_helpers::elixir::{ + elixir_applying_remote_call, elixir_short_circuit_kind, elixir_short_circuit_name, +}; +use crate::metrics::elixir_decisions::{elixir_pair_section, elixir_receive_dispatches}; + +/// Whether a Call spelling `keyword` takes `try`'s sections — `else`, +/// `rescue`, `catch` — in its keyword form: `try` itself, and the +/// method-defining macros, whose body is an implicit `try` +/// (`def f, do: g(), rescue: (e -> e)`). +fn elixir_takes_try_sections(keyword: &str) -> bool { + keyword == "try" || elixir_is_method_macro(keyword) +} + +/// The cost a keyword-form section carries, matching its block form. +enum ElixirSection { + /// `else:`, which adds +1 as the `Else` token does. + Else, + /// `rescue:` / `catch:`, which add +nesting as `RescueBlock` / + /// `CatchBlock` do. + Handler, +} + +/// The section `pair` spells, when its Call takes it (#1574). Only such +/// a Call scores it, because `else:` is an ordinary option name to any +/// other function; a `with`'s `else:` is that construct's clause +/// dispatch, which its own +nesting already paid. +fn elixir_section<'a>( + pair: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> Option { + let (key, call) = elixir_pair_section(pair, code, ancestors)?; + let keyword = elixir_call_keyword(&call, code)?; + match key { + "else" if matches!(keyword, "if" | "unless") || elixir_takes_try_sections(keyword) => { + Some(ElixirSection::Else) + } + "rescue" | "catch" if elixir_takes_try_sections(keyword) => Some(ElixirSection::Handler), + _ => None, + } +} /// Folds an Elixir `BinaryOperator`'s short-circuit operator children /// into the boolean-sequence counter — Elixir has four (`&&`, `||`, /// `and`, `or`). Each word form is keyed to its symbol (see -/// [`compute_booleans_keyed`]): `and` / `or` demand a boolean left -/// operand, but they are the same conjunction / disjunction to read. +/// [`compute_booleans_keyed`] and [`elixir_short_circuit_kind`]). fn compute_elixir_booleans(node: &Node, stats: &mut Stats) { - compute_booleans_keyed(node, stats, |id| match id.into() { - Elixir::AMPAMP | Elixir::And => Some(Elixir::AMPAMP as u16), - Elixir::PIPEPIPE | Elixir::Or => Some(Elixir::PIPEPIPE as u16), - _ => None, + compute_booleans_keyed(node, stats, |id| { + elixir_short_circuit_kind(id).map(|key| key as u16) }); } +/// [`compute_elixir_booleans`] for a short-circuit `Kernel` remote call +/// (`Kernel.||(a, b)`, `Kernel.or(a, b)`), whose operator sits in the +/// call's name rather than among a `binary_operator`'s children. The +/// sequence is keyed on the call's span, as a `binary_operator`'s is on +/// its own, and on the same symbol, so `a or Kernel.||(b, c)` is one run. +fn compute_elixir_remote_booleans<'a>( + op: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, + stats: &mut Stats, +) { + if let Some(remote) = elixir_applying_remote_call(op, code, ancestors) + && let Some(key) = elixir_short_circuit_name(op, code) + { + stats.structural = stats.boolean_seq.eval_based_on_prev( + key as u16, + remote.call.end_byte(), + stats.structural, + ); + } +} + impl Cognitive for ElixirCode { // Elixir control flow is macro-shaped: `if`, `unless`, `case`, - // `cond`, `with`, `for`, `while`, and `try` each surface as a + // `cond`, `with`, `receive`, `for`, `while`, and `try` each surface as a // `Call` node whose `target` Identifier text spells the keyword. // We classify the Call once on entry (raising nesting), then let // the structural `Else` token and `Rescue` / `Catch` blocks inside @@ -39,9 +99,10 @@ impl Cognitive for ElixirCode { // - `if` / `unless` / `for` / `while`: single-branch control flow, // `+nesting`. Their `else` (token `Elixir::Else` inside an // `ElseBlock`) adds `+1` without nesting, matching Java. - // - `case` / `cond` / `with`: switch-/multi-arm, `+nesting` once on - // the container. Individual `stab_clause` arms do NOT add extra - // cost (matches Java `SwitchBlock` / `case:` rule). + // - `case` / `cond` / `with` / `receive`: switch-/multi-arm, + // `+nesting` once on the container. Individual `stab_clause` arms + // do NOT add extra cost (matches Java `SwitchBlock` / `case:` + // rule). // - `try`: `+0` on the container itself — it is a wrapper, and its // `rescue` / `catch` arms surface as `RescueBlock` / `CatchBlock` // and each one adds `+nesting`, matching Java's `CatchClause` @@ -51,7 +112,9 @@ impl Cognitive for ElixirCode { // `FunctionDefinition` rule, but the function depth deliberately // does NOT bump — see the arm comment below for why. // - `AnonymousFunction` (`fn x -> y end`): lambda nesting bumps. - // - `&&` / `||` / `and` / `or`: boolean sequence cost. + // - `&&` / `||` / `and` / `or`: boolean sequence cost, as is a + // `Kernel.&&` / `Kernel.||` / `Kernel.and` / `Kernel.or` remote + // call. // // Limitations: // - `Enum.reduce` / `Enum.map` and friends are higher-order function @@ -66,10 +129,11 @@ impl Cognitive for ElixirCode { fn compute<'a>( node: &Node<'a>, code: &'a [u8], - _ancestors: Ancestors<'a, '_>, + ancestors: Ancestors<'a, '_>, stats: &mut Stats, nesting_map: &mut NestingMap, ) { + // bca: suppress(cyclomatic, halstead) — kind dispatch table; each gated arm is one rule use Elixir as E; let mut nesting = get_nesting_from_map(node, nesting_map); @@ -79,6 +143,15 @@ impl Cognitive for ElixirCode { Some("if" | "unless" | "for" | "while" | "case" | "cond" | "with") => { increase_nesting(stats, &mut nesting); } + // A `case` over the mailbox, paying as one (#1581) when + // it has message clauses; one holding only an `after` + // timeout is a timed wait, the idiomatic sleep, and + // branches on nothing. The `after` adds nothing either + // way: like `case`, the container pays and its arms do + // not, a timeout beside a bare `_ ->` included. + Some("receive") if elixir_receive_dispatches(node, code) => { + increase_nesting(stats, &mut nesting); + } // `try` is intentionally absent: it is a wrapper for // `rescue` / `catch` arms (each of which earns its own // +nesting via `RescueBlock` / `CatchBlock`). Adding the @@ -117,11 +190,38 @@ impl Cognitive for ElixirCode { _ => {} }, // `else` keyword inside an `else_block` (else arm of `if` - // / `unless` / `with` / `try`). Matches Java/Kotlin's - // `Else` rule: +1 without raising nesting. - E::Else => { + // / `unless` / `try`). Matches Java/Kotlin's `Else` rule: +1 + // without raising nesting. + // + // Not a `with`'s: its `else` holds the clauses that dispatch + // over the value a `<-` failed to match, the arms of the + // `case` the construct is in all but spelling, and `case`'s + // arms add nothing past the +nesting its container paid. + // Ruby draws the same line for the `else` of `case/when` + // (#451); the `if` alternative and `try`'s no-exception + // branch, which no container paid for, keep their +1. The + // keyword form `with(…, do: …, else: (…))` scored 1 all + // along; this puts the block form level with it (#1574). + // The `else` token's parents are the `else_block`, then the + // `do_block`, then the Call. + E::Else + if ancestors + .iter(node) + .nth(2) + .is_none_or(|(call, _)| elixir_call_keyword(&call, code) != Some("with")) => + { increment_by_one(stats); } + // The keyword spellings of the sections above, for the + // owners `elixir_section` names: `if(c, do: 1, else: 2)`, + // `try(do: f(), rescue: (…))`, bare or in brackets. A user + // macro's `foo(do: 1, else: 2)` scores nothing here while + // its `foo do 1 else 2 end` scores the `Else` arm above. + E::Pair => match elixir_section(node, code, ancestors) { + Some(ElixirSection::Else) => increment_by_one(stats), + Some(ElixirSection::Handler) => increase_nesting(stats, &mut nesting), + None => {} + }, // `rescue` / `catch` arms of a `try` Call each add +nesting, // matching Java's `CatchClause` treatment. E::RescueBlock | E::CatchBlock => { @@ -143,6 +243,17 @@ impl Cognitive for ElixirCode { E::BinaryOperator | E::BinaryOperator2 | E::BinaryOperator3 => { compute_elixir_booleans(node, stats); } + // A short-circuit `Kernel` remote call applies the operator + // as `a || b` does, so it opens or continues a sequence the + // same way (#1574), keyed on the call's span as a + // `binary_operator`'s operator is keyed on the operator's. + // `Foo.||(a, b)` is an ordinary call and is not matched; the + // token inside a `binary_operator` is the arm above's. + // `Kernel.and` / `Kernel.or` name the operator with an + // `identifier` instead of a token (#1582). + E::AMPAMP | E::PIPEPIPE | E::Identifier => { + compute_elixir_remote_booleans(node, code, ancestors, stats); + } _ => {} } nesting_map.insert(node.id(), nesting); diff --git a/src/metrics/cognitive/word_operator_tests.rs b/src/metrics/cognitive/word_operator_tests.rs index 2c62cc8a1..d0491baf4 100644 --- a/src/metrics/cognitive/word_operator_tests.rs +++ b/src/metrics/cognitive/word_operator_tests.rs @@ -24,11 +24,25 @@ use crate::*; /// `(twin, mixed, words, expected)`: an unmixed expression, the same /// expression with some operators word-spelled, how many word-operator /// tokens `mixed` must carry, and the cognitive sum both must score. +#[cfg(any( + feature = "elixir", + feature = "irules", + feature = "perl", + feature = "php", + feature = "ruby", +))] type Row = (&'static str, &'static str, usize, u64); /// Scores every row inside `wrap`, after proving each mixed fixture /// carries its word operators — a fixture that silently parsed `and` as /// an identifier would otherwise pass while asserting nothing. +#[cfg(any( + feature = "elixir", + feature = "irules", + feature = "perl", + feature = "php", + feature = "ruby", +))] fn assert_rows( path: &str, wrap: impl Fn(&str) -> String, diff --git a/src/metrics/cyclomatic/elixir.rs b/src/metrics/cyclomatic/elixir.rs index 307938773..f7668684b 100644 --- a/src/metrics/cyclomatic/elixir.rs +++ b/src/metrics/cyclomatic/elixir.rs @@ -7,9 +7,10 @@ )] use super::*; +use crate::lang_helpers::elixir::{elixir_applying_operator, elixir_applying_remote_call}; use crate::metrics::elixir_decisions::{ - elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_when_is_guard, - elixir_with_else_dispatches, + elixir_arrow_is_with_clause, elixir_clause_is_decision, elixir_receive_dispatches, + elixir_receive_timeout_clause, elixir_when_is_guard, elixir_with_else_dispatches, }; impl Cyclomatic for ElixirCode { @@ -23,7 +24,7 @@ impl Cyclomatic for ElixirCode { // The split between standard and modified CCN mirrors the C-family // case/switch treatment: per-arm `stab_clause` nodes contribute // standard, while the multi-arm container Calls (`case`/`cond`/ - // `try`) contribute modified. Single-branch keyword Calls + // `try`/`receive`) contribute modified. Single-branch keyword Calls // (`if`/`unless`/`for`/`while`) contribute to both, as do a `with`'s // `<-` clauses and short-circuit booleans (`&&`, `||`, `and`, `or`). fn compute<'a>( @@ -37,9 +38,9 @@ impl Cyclomatic for ElixirCode { match node.kind_id().into() { // Per-arm decisions: each `stab_clause` is one arm of a - // `case`/`cond`/`with`/anonymous-fn body or a `rescue`/ - // `catch` handler. Standard-only: modified counts a - // `case`/`cond`/`try` container Call once, a `with` once + // `case`/`cond`/`with`/`receive`/anonymous-fn body or a + // `rescue`/`catch` handler. Standard-only: modified counts a + // `case`/`cond`/`try`/`receive` container Call once, a `with` once // only when its `else` dispatches (the `"with"` arm below), // and gives a multi-clause `fn` or `reduce` nothing. // @@ -121,18 +122,24 @@ impl Cyclomatic for ElixirCode { // on the same parent. A remote call naming it, // `Kernel.||(a, b)`, applies the macro and decides like // `a || b`; a user module's `Foo.||(a, b)` is an ordinary - // call and does not. ABC and cognitive still score the - // `Kernel` call 0, as they did before #1534. + // call and does not. ABC and cognitive score the `Kernel` + // call through the same predicate (#1574). E::AMPAMP | E::PIPEPIPE | E::And | E::Or - if crate::lang_helpers::elixir::elixir_applying_operator(node, ancestors) - .is_some() - || crate::lang_helpers::elixir::elixir_remote_call_applies( - node, code, ancestors, - ) => + if elixir_applying_operator(node, ancestors).is_some() + || elixir_applying_remote_call(node, code, ancestors).is_some() => { stats.cyclomatic += 1.; stats.cyclomatic_modified += 1.; } + // `Kernel.and(a, b)` / `Kernel.or(a, b)` name the operator + // with an `identifier` rather than a token, so the arm above + // never sees them; they expand the same short-circuit macro + // and decide as `a and b` does (#1582). Only those two names + // are found, so no other identifier passes the guard. + E::Identifier if elixir_applying_remote_call(node, code, ancestors).is_some() => { + stats.cyclomatic += 1.; + stats.cyclomatic_modified += 1.; + } E::Call => { if let Some(target) = node.child_by_field_name("target") && target.kind_id() == E::Identifier @@ -154,6 +161,19 @@ impl Cyclomatic for ElixirCode { "case" | "cond" | "try" => { stats.cyclomatic_modified += 1.; } + // A `receive` is a `case` over the mailbox and + // pays as one (#1581), when it has message + // clauses to pick between: one holding only an + // `after` timeout is a timed wait. Standard pays + // the timeout clause only when a bare `_ ->` + // already took the one free default arm + // (`elixir_clause_is_decision`). + "receive" if elixir_receive_dispatches(node, code) => { + stats.cyclomatic_modified += 1.; + if elixir_receive_timeout_clause(node, code).is_some() { + stats.cyclomatic += 1.; + } + } // A `with`'s decisions are its `<-` clauses, // counted above in both tiers. What modified // collapses is its `else`, the dispatch over diff --git a/src/metrics/elixir_decisions.rs b/src/metrics/elixir_decisions.rs index 03f891fa0..12aa00b37 100644 --- a/src/metrics/elixir_decisions.rs +++ b/src/metrics/elixir_decisions.rs @@ -8,6 +8,7 @@ //! one module. use crate::lang_helpers::elixir::{elixir_call_keyword, elixir_is_method_macro}; +use crate::metrics::abc::is_operand; use crate::{Ancestors, Elixir, Node}; /// Whether `node` is a `binary_operator` applying `operator`. @@ -172,8 +173,6 @@ fn elixir_is_guard_position<'a>( ) -> bool { use Elixir as E; - const ARGUMENTS: &str = "arguments"; - if parent.kind_id() == E::StabClause as u16 { return parent .child_by_field_name("left") @@ -334,6 +333,24 @@ fn elixir_is_first_clause<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> /// `fn`'s first, except the construct's free default clause /// ([`elixir_is_default_clause`]). /// +/// A `receive` gets exactly one free arm, in every spelling. Its +/// `after` clause is the timeout, the path taken when no message clause +/// matched in time, which makes it `receive`'s default arm — the role a +/// bare `_ ->` plays in a `case` (#1581). But a bare `_ ->` in the `do` +/// section is free under `receive` as well, and with one present the +/// timeout is a path of its own: `receive do {:a, y} -> y; _ -> 0 after +/// 10 -> :t end` reaches a matched message, any other message, and an +/// empty mailbox timing out — three paths, the three-clause `case`'s +/// two decisions. So the `after` clause is free only when the `do` +/// section has no bare catch-all ([`elixir_receive_timeout_clause`]). +/// It is never a dispatch in its own right: the compiler accepts +/// exactly one `after` clause, and its head is a timeout value, not a +/// pattern that can fail. No other construct puts a clause there, since +/// `try`'s `after` takes a plain body, so the section is not among +/// those [`DECISION_SECTION_KEYS`] names. +/// The owning Call scores the timeout once; scoring it per clause would +/// repeatedly scan the message section on compiler-invalid multi-timeout input. +/// /// An anonymous fn's head clause is the closure's definition, not a /// dispatch: the closure opens its own function space whose base path /// already covers it, so counting it too would score a trivial @@ -349,8 +366,20 @@ fn elixir_is_first_clause<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> /// [`elixir_when_is_guard`] excludes a typespec's `when` (#1531). The /// keyword form of a clause construct (`case(x, do: (1 -> :a; …))`) /// holds its clauses in the same parenthesised `block`, so a `block` -/// counts when it is the value of a section pair -/// ([`elixir_is_section_pair`]). +/// counts when it is the value of a pair whose key names a clause +/// section ([`DECISION_SECTION_KEYS`]). +/// +/// That pair is not asked which Call owns it, so a clause block passed +/// under a section key to *any* call counts: `f(x, rescue: (e -> e))` +/// scores one decision (#1583). Only a macro can take such an argument +/// — the compiler rejects it for a function ("cannot handle clauses with +/// the -> operator because it is not a macro", Elixir 1.17) — and a +/// user macro's clauses mean whatever it makes of them, so gating on the +/// owner would change no valid program's score in a way either answer +/// could defend. A dispatching DSL is the likelier reading, and is the +/// one kept. Cognitive gates its keyword sections on the owner for a +/// different reason: it scores the pair whatever its value holds, and +/// `else:` is an ordinary option to any function (`f(x, else: 1)`). /// /// Shared by the `Cyclomatic` and `Abc` impls for `ElixirCode`, which /// both score a clause construct per clause and must agree on which @@ -366,16 +395,11 @@ pub(super) fn elixir_clause_is_decision<'a>( let in_section = up.next().is_some_and(|parent| { matches!( parent.kind_id().into(), - E::DoBlock - | E::ElseBlock - | E::AfterBlock - | E::RescueBlock - | E::CatchBlock - | E::AnonymousFunction + E::DoBlock | E::ElseBlock | E::RescueBlock | E::CatchBlock | E::AnonymousFunction ) || (parent.kind_id() == E::Block as u16 - && up - .next() - .is_some_and(|pair| elixir_is_section_pair(&pair, code, SECTION_KEYS))) + && up.next().is_some_and(|pair| { + elixir_pair_key(&pair, code).is_some_and(|key| DECISION_SECTION_KEYS.contains(&key)) + })) }); if !in_section { return false; @@ -397,59 +421,256 @@ pub(super) fn elixir_clause_is_decision<'a>( /// the failed value, which collapses into one decision exactly when it /// holds one — so `else _ -> 0`, which picks nothing, stays free in /// modified as it is in standard, like Rust's `if let … else` (#1535). -/// Spelled in either form: the block form's `else_block`, or the -/// keyword form's `else:` pair whose value is a parenthesised `block`. +/// Spelled in any form: the block form's `else_block`, or the keyword +/// form's `else:` pair whose value is a parenthesised `block`, bare or +/// bracketed ([`elixir_trailing_keywords`]). /// /// In a `with`'s `else` the default-clause rule reduces to the bare /// `_`: `true ->` is free only under a `cond`, and the head skip only /// under a `fn` or `reduce:`, so [`elixir_clause_is_decision`] agrees /// clause by clause with the test below. pub(super) fn elixir_with_else_dispatches(call: &Node, code: &[u8]) -> bool { - use Elixir as E; + elixir_section_has_clause(call, code, Section::Else, |clause| { + !elixir_is_catch_all(clause, code) + }) +} + +/// Whether `clause` (a `stab_clause`) is a bare, unguarded `_ ->`. +fn elixir_is_catch_all(clause: &Node, code: &[u8]) -> bool { + elixir_sole_unguarded_pattern(clause) + .is_some_and(|pattern| elixir_is_bare_wildcard(&pattern, code)) +} + +/// A `receive`'s `after` clause when it is a decision: when its `do` +/// section already holds the free bare `_ ->`, leaving the timeout as a +/// path of its own. Without one the timeout is the construct's single +/// free arm ([`elixir_clause_is_decision`]). +/// Called once per receive, not once per timeout clause. +pub(super) fn elixir_receive_timeout_clause<'a>( + receive: &Node<'a>, + code: &'a [u8], +) -> Option> { + let timeout = receive.children().find_map(|child| { + elixir_section_holder(&child, code, Section::After)? + .children() + .find(|clause| clause.kind_id() == Elixir::StabClause as u16) + })?; + elixir_section_has_clause(receive, code, Section::Do, |clause| { + elixir_is_catch_all(clause, code) + }) + .then_some(timeout) +} + +/// Whether a `receive` Call dispatches over the mailbox: whether its +/// `do` section holds a message clause, in any spelling. One holding +/// only an `after` timeout (`receive do after 100 -> :ok end`, the +/// idiomatic sleep) is a timed wait that picks between nothing, so it +/// pays neither modified cyclomatic's container decision nor cognitive's +/// +nesting — as a `with` whose `else` picks nothing pays no modified +/// decision ([`elixir_with_else_dispatches`]). Any clause counts, a bare +/// `_ ->` included, because `case x do _ -> 0 end` pays its container +/// too (#1581). +pub(super) fn elixir_receive_dispatches(call: &Node, code: &[u8]) -> bool { + elixir_section_has_clause(call, code, Section::Do, |_| true) +} + +/// A section of a Call, as [`elixir_section_holder`] finds it. +#[derive(Clone, Copy)] +pub(super) enum Section { + Do, + Else, + After, +} +impl Section { + /// The section's keyword-form spelling, without its colon. + const fn key(self) -> &'static str { + match self { + Self::Do => "do", + Self::Else => "else", + Self::After => "after", + } + } +} + +/// Whether `call`'s `section` holds a `stab_clause` that `accept`s. +fn elixir_section_has_clause( + call: &Node, + code: &[u8], + section: Section, + accept: impl Fn(&Node) -> bool, +) -> bool { call.children().any(|child| { - let section = match child.kind_id().into() { - E::DoBlock => child - .children() - .find(|c| c.kind_id() == E::ElseBlock as u16), - _ if child.kind() == "arguments" => child + elixir_section_holder(&child, code, section).is_some_and(|holder| { + holder .children() - .filter(|c| c.kind_id() == E::Keywords as u16) - .flat_map(|keywords| keywords.children()) - .find(|pair| elixir_is_section_pair(pair, code, &["else:"])) - .and_then(|pair| pair.child_by_field_name("value")), - _ => None, - }; - section.is_some_and(|section| { - section.children().any(|clause| { - clause.kind_id() == E::StabClause as u16 - && !elixir_sole_unguarded_pattern(&clause) - .is_some_and(|pattern| elixir_is_bare_wildcard(&pattern, code)) - }) + .any(|clause| clause.kind_id() == Elixir::StabClause as u16 && accept(&clause)) }) }) } -/// The keyword spellings of the clause sections a `do … end` block -/// spells as `do_block` / `else_block` / `after_block` / `rescue_block` -/// / `catch_block`. -const SECTION_KEYS: &[&str] = &["do:", "else:", "after:", "rescue:", "catch:"]; - -/// Whether `pair` is a keyword-form section (`do: (…)`) whose key is one -/// of `keys`. The key token's text carries its colon and the whitespace -/// after it, so it is trimmed before comparing. -fn elixir_is_section_pair(pair: &Node, code: &[u8], keys: &[&str]) -> bool { - pair.kind_id() == Elixir::Pair as u16 - && pair - .child_by_field_name("key") - .and_then(|key| key.utf8_text(code)) - .is_some_and(|key| keys.contains(&key.trim_end())) +/// Whether `call` has a `section`, in either spelling +/// ([`elixir_section_holder`]). +pub(super) fn elixir_call_has_section(call: &Node, code: &[u8], section: Section) -> bool { + call.children() + .any(|child| elixir_section_holder(&child, code, section).is_some()) +} + +/// The node holding the `section` that `child`, a child of a Call, +/// spells: the block form's `do_block`, or the `else_block` inside it, +/// or the keyword form's `do:` / `else:` value in the trailing keyword +/// list of the `arguments` `child` is, bare or bracketed +/// ([`elixir_trailing_keywords`]). The block form's `else_block` is the +/// `do_block`'s child, never the Call's. +fn elixir_section_holder<'a>( + child: &Node<'a>, + code: &'a [u8], + section: Section, +) -> Option> { + use Elixir as E; + + match (child.kind_id().into(), section) { + (E::DoBlock, Section::Do) => Some(*child), + (E::DoBlock, Section::Else) => child + .children() + .find(|c| c.kind_id() == E::ElseBlock as u16), + (E::DoBlock, Section::After) => child + .children() + .find(|c| c.kind_id() == E::AfterBlock as u16), + _ if child.kind() == ARGUMENTS => elixir_keyword_section(child, code, section.key()), + _ => None, + } } -/// The `Call` whose `do` section holds `node` (a `stab_clause`): the -/// block form's `do_block` (whose parent is the call) or the keyword -/// form's `do: (…)` block (`block` → `pair` → `keywords` → `arguments` -/// → call). The caller asks the Call's keyword (`cond`, `for`). +/// The keyword spellings of the clause sections whose clauses are +/// decisions, which a `do … end` block spells as `do_block` / +/// `else_block` / `rescue_block` / `catch_block`. Not `after`: see +/// [`elixir_clause_is_decision`]. +const DECISION_SECTION_KEYS: &[&str] = &["do", "else", "rescue", "catch"]; + +/// The key a keyword-list `pair` spells, without its colon: `do` for +/// both `do: x` and the quoted `"do": x`, which Elixir reads as the same +/// atom. The key token's text carries its colon and the whitespace after +/// it. `None` for any other node, and for a quoted key that interpolates +/// (`"do#{k}": x`), whose atom is not known until run time — valid in an +/// ordinary call's options, but never a section. +/// +/// The grammar's key is a `keyword` token, which has no named parts, or a +/// `quoted_keyword` around its `quoted_content` and any interpolations. +fn elixir_pair_key<'a>(pair: &Node<'a>, code: &'a [u8]) -> Option<&'a str> { + if pair.kind_id() != Elixir::Pair as u16 { + return None; + } + let key = pair.child_by_field_name("key")?; + let mut parts = key.children().filter(Node::is_named); + match (parts.next(), parts.next()) { + (None, _) => key + .utf8_text(code) + .and_then(|text| text.trim_end().strip_suffix(':')), + (Some(content), None) if content.kind() == "quoted_content" => content.utf8_text(code), + _ => None, + } +} + +/// The keyword list `arguments` ends with: a bare `keywords` node +/// (`if c, do: x`), or the same list in brackets (`if(c, [do: x])`), +/// which Elixir reads as the identical trailing keyword list — the +/// keyword syntax is sugar for exactly that last-argument literal. +/// +/// Only the *last* argument qualifies, and only a list holding nothing +/// but the `keywords` node: `[a: 1]` in any other position is an +/// ordinary list argument, and `[1, do: x]` is a list with a keyword +/// tail, not a keyword list. An empty `[]` holds no `keywords` at all. +/// The grammar ends a list with its `keywords`, so a list whose first +/// item is one holds nothing else. +/// +/// Matched by rule name: `keywords` carries two kind ids at this pin +/// (grammar-dispatch §1). Comments are skipped as the extras they are. +fn elixir_trailing_keywords<'a>(arguments: &Node<'a>) -> Option> { + let last = arguments.children().filter(is_operand).last()?; + if last.kind() == KEYWORDS { + return Some(last); + } + if last.kind_id() != Elixir::List as u16 { + return None; + } + last.children() + .find(is_operand) + .filter(|keywords| keywords.kind() == KEYWORDS) +} + +/// The value of the `key:` pair in `arguments`' trailing keyword list +/// ([`elixir_trailing_keywords`]), or `None` when it has no such pair. +fn elixir_keyword_section<'a>(arguments: &Node<'a>, code: &'a [u8], key: &str) -> Option> { + elixir_trailing_keywords(arguments)? + .children() + .find(|pair| elixir_pair_key(pair, code) == Some(key))? + .child_by_field_name("value") +} + +/// The section key of `pair` and the `Call` whose trailing keyword list +/// holds it — `("else", if …)` for the `else:` of `if(c, do: 1, else: 2)` +/// or of `if(c, [do: 1, else: 2])` — or `None` when `pair` is not a +/// keyword argument of a Call. `ancestors` is `pair`'s chain. +/// +/// The climb is `keywords`, an optional bracketing `list`, `arguments`, +/// then the Call, and the keyword list must be the trailing one +/// [`elixir_trailing_keywords`] describes: no operand follows it among +/// the arguments, and a bracketing list holds nothing before it. So a +/// `[do: x]` in any other argument position, a map's keyword pairs and a +/// list nested in a list (`f([[do: x]])`, whose outer list is where the +/// `arguments` should be) are not sections. +/// +/// Each step reads one sibling off the chain rather than the argument +/// list from its front, so the pairs of one call cost it a scan each +/// only once: every pair asks this in cognitive, and every keyword-form +/// clause in cyclomatic and ABC, and a call holding thousands of +/// positional arguments beside thousands of pairs or clauses was +/// quadratic in them (5.4 s for 16 000 arguments in a release build; +/// `cognitive/elixir-mixed-arguments` prices the shape). +pub(super) fn elixir_pair_section<'a>( + pair: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, +) -> Option<(&'a str, Node<'a>)> { + let mut up = ancestors.iter(pair); + let (keywords, keywords_ancestors) = up.next().filter(|(node, _)| node.kind() == KEYWORDS)?; + // The argument the keyword list is: itself, or the list bracketing it. + let (bracketed, (argument, argument_ancestors)) = match up.next()? { + (list, list_ancestors) if list.kind_id() == Elixir::List as u16 => { + (true, (list, list_ancestors)) + } + _ => (false, (keywords, keywords_ancestors)), + }; + let mut above = argument_ancestors.iter(&argument); + let (arguments, _) = above.next()?; + let (call, _) = above.next()?; + // The grammar ends a list with its `keywords`, so a bracketing list + // that holds nothing before them holds nothing else. + let is_section = arguments.kind() == ARGUMENTS + && call.kind_id() == Elixir::Call as u16 + && argument_ancestors + .siblings_after(&argument) + .all(|sibling| !is_operand(&sibling)) + && (!bracketed + || keywords_ancestors + .siblings_before(&keywords) + .all(|sibling| !is_operand(&sibling))); + is_section + .then(|| elixir_pair_key(pair, code)) + .flatten() + .map(|key| (key, call)) +} + +/// The rule name of a keyword list (two kind ids at this pin). +const KEYWORDS: &str = "keywords"; + +/// The rule name of an argument list (five kind ids at this pin). +const ARGUMENTS: &str = "arguments"; + +/// The `Call` whose `do` section holds `node` (a `stab_clause`), in +/// either spelling: the block form's `do_block`, or a trailing `do:` +/// keyword pair. The caller asks the Call's keyword (`cond`, `for`). fn elixir_do_section_call<'a>( node: &Node<'a>, code: &'a [u8], @@ -457,13 +678,64 @@ fn elixir_do_section_call<'a>( ) -> Option> { use Elixir as E; - let mut up = ancestors.iter(node).map(|(ancestor, _)| ancestor); - match up.next().map(|parent| parent.kind_id().into()) { - Some(E::DoBlock) => up.next(), - Some(E::Block) => up - .next() - .filter(|pair| elixir_is_section_pair(pair, code, &["do:"])) - .and_then(|_| up.nth(2)), + let mut up = ancestors.iter(node); + let (holder, _) = up.next()?; + let (next, next_ancestors) = up.next()?; + match holder.kind_id().into() { + E::DoBlock => Some(next), + E::Block => elixir_pair_section(&next, code, next_ancestors) + .filter(|(key, _)| *key == "do") + .map(|(_, call)| call), _ => None, } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ElixirCode; + use crate::test_support::for_each_node_with_chain; + + #[cfg(feature = "elixir")] + #[test] + fn empty_call_arguments_have_no_trailing_keywords() { + let source = b"f()\ng(else: 1)\n"; + let mut found = Vec::new(); + for_each_node_with_chain::(source, |node, _| { + if node.kind() == ARGUMENTS { + found.push(elixir_trailing_keywords(node).map(|keywords| { + keywords + .utf8_text(source) + .expect("keywords are UTF-8") + .to_owned() + })); + } + }); + assert_eq!(found, [None, Some("else: 1".to_owned())]); + } + + #[cfg(feature = "elixir")] + #[test] + fn standalone_keyword_list_has_no_owning_call() { + let source = b"[else: 1]\nf(else: 2)\n"; + let mut found = Vec::new(); + for_each_node_with_chain::(source, |node, chain| { + if node.kind_id() == Elixir::Pair as u16 { + found.push( + elixir_pair_section(node, source, Ancestors::known(chain)).map( + |(key, call)| { + ( + key.to_owned(), + call.utf8_text(source).expect("call is UTF-8").to_owned(), + ) + }, + ), + ); + } + }); + assert_eq!( + found, + [None, Some(("else".to_owned(), "f(else: 2)".to_owned()))] + ); + } +} diff --git a/src/metrics/elixir_spelling_parity_tests.rs b/src/metrics/elixir_spelling_parity_tests.rs new file mode 100644 index 000000000..f2337b663 --- /dev/null +++ b/src/metrics/elixir_spelling_parity_tests.rs @@ -0,0 +1,443 @@ +//! Two spellings of one Elixir program score alike (#1574). +//! +//! Elixir writes a construct's sections three ways: a `do … end` block, a +//! trailing keyword list (`if c, do: 1, else: 2`), and the same list in +//! brackets (`if(c, [do: 1, else: 2])`), which the compiler reads as the +//! identical last argument. And it applies a short-circuit operator either +//! as an operator (`a || b`) or as a `Kernel` remote call +//! (`Kernel.||(a, b)`), which expands the same macro. +//! +//! Every group pairs the spellings with a twin whose score predates the +//! fix — the block form, or the operator form — and that is the oracle: +//! the twin's numbers are asserted outright, so a group cannot pass by +//! every spelling drifting together. The one twin that moved is the +//! `with … else` block form's cognitive score, decided in +//! [`elixir_with_else_scores_like_case`] against `case`. + +use crate::test_support::metrics_verbatim; +use crate::*; + +/// `[cyclomatic, cyclomatic_modified, abc conditions, cognitive]` sums of +/// the root space; both cyclomatic tiers carry a base of 1. +#[cfg(feature = "elixir")] +type Scores = [u64; 4]; + +#[cfg(feature = "elixir")] +fn measure(source: &str) -> Scores { + let m = metrics_verbatim( + LANG::Elixir, + format!("{source}\n").as_bytes(), + MetricsOptions::default().with_only(&[Metric::Cyclomatic, Metric::Abc, Metric::Cognitive]), + ); + [ + m.cyclomatic.cyclomatic_sum(), + m.cyclomatic.cyclomatic_modified_sum(), + m.abc.conditions_sum(), + m.cognitive.cognitive_sum(), + ] +} + +/// Asserts `twin` scores `want`, then that every spelling scores the same. +#[cfg(feature = "elixir")] +fn assert_group(twin: &str, want: Scores, spellings: &[&str]) { + assert_eq!( + measure(twin), + want, + "twin `{twin}` moved; re-derive the group" + ); + for spelling in spellings { + assert_eq!( + measure(spelling), + want, + "`{spelling}` must score like `{twin}`" + ); + } +} + +/// A `Kernel` remote call naming `&&` / `||` is the operator: one +/// decision, one ABC slot per operand, one cognitive sequence. Before +/// #1574 only cyclomatic counted it; ABC and cognitive scored 0, and a +/// nested call read as a plain operand, so `a || Kernel.||(b, c)` paid a +/// slot for the inner call that `a || (b || c)` does not. +#[cfg(feature = "elixir")] +#[test] +fn elixir_kernel_remote_call_scores_like_the_operator() { + assert_group( + "a || b", + [2, 2, 2, 1], + &[ + "Kernel.||(a, b)", + "Elixir.Kernel.||(a, b)", + // The pipe passes the piped value as the first operand. + "a |> Kernel.||(b)", + ], + ); + assert_group("a && b", [2, 2, 2, 1], &["Kernel.&&(a, b)"]); + // A nested chain pays its own operands and no slot as an operand. + assert_group( + "a || (b || c)", + [3, 3, 3, 1], + &[ + "Kernel.||(a, Kernel.||(b, c))", + "a || Kernel.||(b, c)", + "Kernel.||(a, b || c)", + ], + ); + // A change of operator opens a second cognitive sequence. + assert_group( + "a && (b || c)", + [3, 3, 3, 2], + &["Kernel.&&(a, Kernel.||(b, c))"], + ); + // Piping the call's *result* passes no operand to it. + assert_group("(a || b) |> f()", [2, 2, 2, 1], &["Kernel.||(a, b) |> f()"]); + // As an `if` predicate the call scores itself, as the chain does. + assert_group( + "if a || b, do: 1", + [3, 3, 2, 2], + &["if Kernel.||(a, b), do: 1"], + ); +} + +/// `Kernel.and` / `Kernel.or` expand the same short-circuit macros as +/// `a and b` / `a or b`, but the grammar names them with a plain +/// `identifier` rather than an operator token, so before #1582 they +/// scored nothing in any metric. Each word form keys the cognitive +/// sequence its symbol does, so a mixed run of `or` and `||` is one +/// sequence and a change of operation opens a second. +#[cfg(feature = "elixir")] +#[test] +fn elixir_kernel_word_operator_scores_like_the_operator() { + assert_group( + "a and b", + [2, 2, 2, 1], + &[ + "Kernel.and(a, b)", + "Elixir.Kernel.and(a, b)", + "a |> Kernel.and(b)", + ], + ); + assert_group("a or b", [2, 2, 2, 1], &["Kernel.or(a, b)"]); + assert_group( + "a or (b or c)", + [3, 3, 3, 1], + &["Kernel.or(a, Kernel.or(b, c))", "a or Kernel.||(b, c)"], + ); + assert_group( + "a and (b or c)", + [3, 3, 3, 2], + &["Kernel.and(a, Kernel.or(b, c))"], + ); + assert_group( + "if a and b, do: 1", + [3, 3, 2, 2], + &["if Kernel.and(a, b), do: 1"], + ); +} + +/// A negation decides nothing in any spelling, applied or called on +/// `Kernel`; a user module's `and/2` is an ordinary function; and a +/// capture or a bare reference only names `Kernel`'s. +#[cfg(feature = "elixir")] +#[test] +fn elixir_kernel_negation_and_foreign_word_operator_decide_nothing() { + assert_group( + "not x", + [1, 1, 0, 0], + &["!x", "Kernel.not(x)", "Kernel.!(x)"], + ); + assert_group( + "Foo.foo(a, b)", + [1, 1, 0, 0], + &[ + "Foo.and(a, b)", + "Foo.or(a, b)", + "&Kernel.and/2", + "Kernel.and", + ], + ); +} + +/// A user module's `||/2` is an ordinary function, and a capture only +/// names `Kernel`'s: neither decides anything in any metric. +#[cfg(feature = "elixir")] +#[test] +fn elixir_non_kernel_remote_call_decides_nothing() { + assert_group( + "Foo.foo(a, b)", + [1, 1, 0, 0], + &["Foo.||(a, b)", "a |> Foo.||(b)", "&Kernel.||/2"], + ); + // In a slot, neither a user module's `||` nor a `Kernel` function + // that is no operator scores itself: each pays the slot as a call. + assert_group( + "if foo(a), do: 1", + [2, 2, 1, 1], + &["if Kernel.foo(a), do: 1", "if Foo.||(a, b), do: 1"], + ); +} + +/// `if` / `unless` with an `else`, in every spelling of the keyword list, +/// including a trailing comma and quoted keys (`"do":` is the atom +/// `:do`). Before #1574 the keyword forms lost cognitive's `else` +1, +/// and the bracketed and quoted forms lost ABC's `else` condition too. +#[cfg(feature = "elixir")] +#[test] +fn elixir_if_else_scores_alike_in_every_spelling() { + assert_group( + "if c do 1 else 2 end", + [2, 2, 2, 2], + &[ + "if(c, do: 1, else: 2)", + "if(c, [do: 1, else: 2])", + "if(c, [do: 1, else: 2,])", + "if(c, \"do\": 1, \"else\": 2)", + "if(c, [\"do\": 1, \"else\": 2])", + ], + ); + assert_group( + "unless c do 1 else 2 end", + [2, 2, 2, 2], + &["unless(c, do: 1, else: 2)", "unless(c, [do: 1, else: 2])"], + ); + // Each `else` belongs to its own `if`, however the two nest. + assert_group( + "if a do (if b do 1 else 2 end) else 3 end", + [3, 3, 4, 5], + &[ + "if(a, do: if(b, do: 1, else: 2), else: 3)", + "if(a, [do: if(b, [do: 1, else: 2]), else: 3])", + ], + ); +} + +/// Only the *last* argument, and only a list holding nothing but pairs, +/// is a keyword list: `[do: 1, else: 2]` in first position, `[1, do: …]` +/// and `[]` are ordinary list arguments, so the `if` has no `else`. And +/// `else:` is an ordinary option name to any other function. +#[cfg(feature = "elixir")] +#[test] +fn elixir_list_that_is_not_a_keyword_list_has_no_sections() { + assert_group("[1]", [1, 1, 0, 0], &["[else: 1]", "[do: 1, else: 2]"]); + assert_group( + "if(c, x)", + [2, 2, 1, 1], + &[ + "if([do: 1, else: 2], c)", + "if(c, [1, do: 1, else: 2])", + "if(c, [])", + ], + ); + assert_group( + "f(x, y)", + [1, 1, 0, 0], + &["f(x, else: 1)", "f(x, [else: 1])"], + ); + assert_group( + "Foo.bar(x, y)", + [1, 1, 0, 0], + &["Foo.bar(x, else: 1)", "Foo.bar(x, [else: 1])"], + ); + // An interpolated key has no static name. Only an ordinary call + // takes one validly, and no ordinary call has sections, so this row + // exercises the reading rather than discriminating it. + assert_group("f(x, y)", [1, 1, 0, 0], &["f(x, \"else#{y}\": 1)"]); +} + +/// Clause constructs: the bracketed `cond` lost its free `true ->` +/// default, because the `cond` lookup climbed a fixed number of parents. +#[cfg(feature = "elixir")] +#[test] +fn elixir_clause_constructs_score_alike_in_every_spelling() { + assert_group( + "case x do 1 -> :a; 2 -> :b end", + [3, 2, 2, 1], + &[ + "case(x, do: (1 -> :a; 2 -> :b))", + "case(x, [do: (1 -> :a; 2 -> :b)])", + ], + ); + assert_group( + "cond do x > 1 -> :a; true -> :b end", + [2, 2, 1, 1], + &[ + "cond(do: (x > 1 -> :a; true -> :b))", + "cond([do: (x > 1 -> :a; true -> :b)])", + ], + ); +} + +/// `try`'s `rescue:` / `catch:` pay +nesting and its `else:` +1 in the +/// keyword forms, as their blocks do; before #1574 the keyword `try` +/// scored cognitive 0. A `def` body is an implicit `try` and takes the +/// same sections. +#[cfg(feature = "elixir")] +#[test] +fn elixir_try_sections_score_alike_in_every_spelling() { + assert_group( + "try do f() rescue e in RuntimeError -> e catch :x -> 1 end", + [3, 2, 2, 2], + &[ + "try(do: f(), rescue: (e in RuntimeError -> e), catch: (:x -> 1))", + "try([do: f(), rescue: (e in RuntimeError -> e), catch: (:x -> 1)])", + ], + ); + assert_group( + "try do f() rescue _ -> 1 else x -> x end", + [2, 2, 1, 2], + &[ + "try(do: f(), rescue: (_ -> 1), else: (x -> x))", + "try([do: f(), rescue: (_ -> 1), else: (x -> x)])", + ], + ); + // A handler raises nesting, not just +1: the `if` inside it costs 2. + assert_group( + "try do f() rescue _ -> (if a do 1 end) end", + [2, 3, 1, 3], + &[ + "try(do: f(), rescue: (_ -> if(a, do: 1)))", + "try([do: f(), rescue: (_ -> if(a, do: 1))])", + ], + ); + assert_group( + "try do f() catch _ -> (if a do 1 end) end", + [2, 3, 1, 3], + &["try(do: f(), catch: (_ -> if(a, do: 1)))"], + ); + assert_group( + "def f() do\n g()\nrescue\n e -> e\nend", + [3, 2, 1, 1], + &["def f(), do: g(), rescue: (e -> e)"], + ); +} + +/// A `with`'s `else` holds the clauses that dispatch over the value a +/// `<-` failed to match: the arms of a `case` in all but spelling. A +/// `case`'s arms add no cognitive cost past the +nesting its container +/// pays, and Ruby draws the same line for the `else` of `case/when` +/// (#451), so the block form's `else` token adds nothing either; the +/// keyword form already scored that (#1574). An `if`'s `else` is a +/// conditional's alternative and keeps its +1 (see the nested row). +/// +/// The bracketed form also lost its modified tier: the `else` lookup +/// read only a bare keyword list, so the dispatching `else` looked absent. +#[cfg(feature = "elixir")] +#[test] +fn elixir_with_else_scores_like_case() { + // The oracle: the same dispatch as one `case`, cognitive 1. + assert_eq!( + measure("case g(x) do {:ok, a} -> a; {:error, e} -> e; :z -> 0 end")[3], + 1 + ); + assert_group( + "with({:ok, a} <- g(x), do: a, else: ({:error, e} -> e; :z -> 0))", + [4, 3, 3, 1], + &[ + "with {:ok, a} <- g(x) do a else {:error, e} -> e; :z -> 0 end", + "with({:ok, a} <- g(x), [do: a, else: ({:error, e} -> e; :z -> 0)])", + ], + ); + // A non-dispatching `else _ -> 0` is free in the modified tier too. + assert_group( + "with({:ok, a} <- g(x), do: a, else: (_ -> 0))", + [2, 2, 1, 1], + &[ + "with {:ok, a} <- g(x) do a else _ -> 0 end", + "with({:ok, a} <- g(x), [do: a, else: (_ -> 0)])", + ], + ); + // The `if` nested in the body keeps its own `else` +1 at nesting 1. + assert_group( + "with({:ok, a} <- g(x), do: if(a, do: 1, else: 2), else: (_ -> 0))", + [3, 3, 3, 4], + &["with {:ok, a} <- g(x) do (if a do 1 else 2 end) else _ -> 0 end"], + ); +} + +/// A `for … reduce:`'s first accumulator clause is its head, free in +/// every spelling (#1535). The bracketed form missed the `for` behind +/// the list and scored the head as a decision. +#[cfg(feature = "elixir")] +#[test] +fn elixir_for_reduce_head_is_free_in_every_spelling() { + assert_group( + "for x <- xs, reduce: 0 do acc -> acc + x end", + [2, 2, 0, 1], + &[ + "for(x <- xs, reduce: 0, do: (acc -> acc + x))", + "for(x <- xs, [reduce: 0, do: (acc -> acc + x)])", + ], + ); + assert_group( + "for x <- xs, reduce: 0 do 0 -> x; acc -> acc + x end", + [3, 2, 1, 1], + &[ + "for(x <- xs, reduce: 0, do: (0 -> x; acc -> acc + x))", + "for(x <- xs, [reduce: 0, do: (0 -> x; acc -> acc + x)])", + ], + ); +} + +/// A `receive` is a `case` over the mailbox and scores as one in every +/// spelling (#1581): before, modified cyclomatic and cognitive did not +/// know the construct, so it sat at the base of both, and its `after` +/// timeout counted as one more clause. The timeout is the path taken +/// when no clause matched in time — `receive`'s default arm, which +/// `case`'s `_ ->` twin shows is free — so with it the `receive` still +/// scores what the two-clause `case` does. A `receive` has one free arm, +/// though, so beside a bare `_ ->` the timeout counts. +#[cfg(feature = "elixir")] +#[test] +fn elixir_receive_scores_like_case() { + assert_group( + "case x do {:a, y} -> y; :b -> 0 end", + [3, 2, 2, 1], + &[ + "receive do {:a, y} -> y; :b -> 0 end", + "receive do {:a, y} -> y; :b -> 0 after 10 -> :t end", + "receive(do: ({:a, y} -> y; :b -> 0), after: (10 -> :t))", + "receive([do: ({:a, y} -> y; :b -> 0), after: (10 -> :t)])", + ], + ); + // The container raises nesting for what its clauses hold. + assert_group( + "case x do {:a, y} -> (if y do 1 end) end", + [3, 3, 2, 3], + &[ + "receive do {:a, y} -> (if y do 1 end) after 10 -> :t end", + "receive(do: ({:a, y} -> if(y, do: 1)), after: (10 -> :t))", + ], + ); + // A `receive` gets one free arm. With a bare `_ ->` in its `do` + // section the timeout is a path of its own: a matched `{:a, y}`, any + // other message, and an empty mailbox timing out make three paths, + // so cyclomatic is 3 (two decisions: `{:a, y}` and `after`, the `_` + // free), modified 2 (base + the container), C 2 (one per decision), + // cognitive 1 (+nesting on the container, arms free) — exactly the + // three-clause `case` beside it, whose `_ ->` is its free arm. + assert_group( + "case x do {:a, y} -> y; :b -> 0; _ -> 1 end", + [3, 2, 2, 1], + &[ + "receive do {:a, y} -> y; _ -> 0 after 10 -> :t end", + "receive(do: ({:a, y} -> y; _ -> 0), after: (10 -> :t))", + "receive([do: ({:a, y} -> y; _ -> 0), after: (10 -> :t)])", + ], + ); + // A lone catch-all is free in standard and still pays the container. + assert_group( + "case x do _ -> 0 end", + [1, 2, 0, 1], + &["receive do _ -> 0 end", "receive(do: (_ -> 0))"], + ); + // With no message clause it is a timed wait, the sleep it spells. + assert_group( + "Process.sleep(10)", + [1, 1, 0, 0], + &[ + "receive do after 10 -> :t end", + "receive(after: (10 -> :t))", + "receive([after: (10 -> :t)])", + ], + ); +} diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index a54cfb586..25c6f9a50 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -602,10 +602,14 @@ mod tests { /// among the operands would pass on that. The operator side is what /// pins the removal. #[cfg(any( + feature = "cpp", feature = "csharp", feature = "groovy", feature = "java", - feature = "kotlin" + feature = "kotlin", + feature = "mozcpp", + feature = "objc", + feature = "typescript" ))] #[track_caller] fn assert_keywords_are_operands_only( @@ -2683,27 +2687,30 @@ mod tests { }", "foo.java", |metric| { - // Operators (n1=12): class {} void () [] , . ; int = + / + // Operators (n1=14): public static class {} void () [] , . ; int = + / + // (`public` x2 and `static` since #1565) // Operands (n2=13): Main main string args a b c avg 5 3 MessageFormat format "{0}" // (`string` is the parameter's type name, an operand since #1560) + assert_eq!(metric.halstead.unique_operators(), 14); + assert_eq!(metric.halstead.total_operators(), 30); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 12, - "total_operators": 27, + "unique_operators": 14, + "total_operators": 30, "unique_operands": 13, "total_operands": 23, - "length": 50, - "estimated_program_length": 91.12526634448807, - "purity_ratio": 1.8225053268897613, - "vocabulary": 25, - "volume": 232.19280948873623, - "difficulty": 10.615384615384615, - "level": 0.09420289855072464, - "effort": 2464.8159776496614, - "time": 136.93422098053674, - "bugs": 0.06082308053869885 + "length": 53, + "estimated_program_length": 101.40868524464065, + "purity_ratio": 1.913371419710201, + "vocabulary": 27, + "volume": 252.0090376146638, + "difficulty": 12.384615384615385, + "level": 0.08074534161490683, + "effort": 3121.035004304683, + "time": 173.3908335724824, + "bugs": 0.07118871793971378 } "# ); @@ -2730,26 +2737,27 @@ mod tests { |metric| { // Verifies all 8 Java primitive-type keywords (byte, short, int, long, // char, float, double, boolean) are counted as distinct operators, and - // that true/false are counted as operands. The other four operators - // are `class` (#1552), `{}`, `=` and `;`. + // that true/false are counted as operands. The other five operators + // are `public` (#1565), `class` (#1552), `{}`, `=` and `;`. + assert_eq!(metric.halstead.unique_operators(), 13); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 12, - "total_operators": 29, + "unique_operators": 13, + "total_operators": 30, "unique_operands": 19, "total_operands": 19, - "length": 48, - "estimated_program_length": 123.730172764082, - "purity_ratio": 2.5777119325850415, - "vocabulary": 31, - "volume": 237.80142289857, - "difficulty": 6.0, - "level": 0.16666666666666666, - "effort": 1426.80853739142, - "time": 79.26714096619, - "bugs": 0.04224635182363915 + "length": 49, + "estimated_program_length": 128.8163390912623, + "purity_ratio": 2.6289048794135166, + "vocabulary": 32, + "volume": 245.0, + "difficulty": 6.5, + "level": 0.15384615384615385, + "effort": 1592.5, + "time": 88.47222222222223, + "bugs": 0.04545674902336332 } "# ); @@ -2941,19 +2949,15 @@ mod tests { // place of Java's `MessageFormat.format(...)`. amaanq's // grammar inherits Java's tokenisation, so n1/N1/n2/N2 // shapes match Java up to those substitutions. - // The dekobon grammar parses primitive type names - // (`void`, `int`, `String`) as `type_identifier` - // rather than as distinct keyword tokens, so they - // count as operands here — the prior amaanq grammar - // treated them as operators. Net shift: −2 unique - // operators (`void`, `int`), +2 unique operands - // (`void`, `int` were the only two type_identifiers - // not already counted as operands, since `String` - // was already an identifier in the prior grammar's - // counting). - // `class` is the ninth unique operator since #1552. - assert_eq!(metric.halstead.unique_operators(), 9); - assert_eq!(metric.halstead.unique_operands(), 13); + // The dekobon grammar parses the primitive type names + // `void` and `int` as `type_identifier`, the kind of + // `String`; since #1568 their spelling bills them as + // operators, as Java's keyword tokens are, while + // `String` stays an operand. + // `class` is an operator since #1552, `static` since + // #1565, and `void` and `int` since #1568. + assert_eq!(metric.halstead.unique_operators(), 12); + assert_eq!(metric.halstead.unique_operands(), 11); // `bugs` is `effort.powf(2/3) / 3000`, and the platform libm's // `powf` differs in the last ulp: macOS rendered this value // as `…92342184` where Linux and Windows gave `…923421845`. @@ -2963,20 +2967,20 @@ mod tests { { ".bugs" => insta::rounded_redaction(12) }, @r#" { - "unique_operators": 9, - "total_operators": 23, - "unique_operands": 13, - "total_operands": 23, - "length": 46, - "estimated_program_length": 76.63504134881501, - "purity_ratio": 1.665979159756848, - "vocabulary": 22, - "volume": 205.13385445731566, - "difficulty": 7.961538461538462, - "level": 0.12560386473429952, - "effort": 1633.1810720255517, - "time": 90.73228177919731, - "bugs": 0.046227630923 + "unique_operators": 12, + "total_operators": 26, + "unique_operands": 11, + "total_operands": 21, + "length": 47, + "estimated_program_length": 81.07329781366414, + "purity_ratio": 1.7249637832694498, + "vocabulary": 23, + "volume": 212.60741193467962, + "difficulty": 11.454545454545455, + "level": 0.0873015873015873, + "effort": 2435.3212639790577, + "time": 135.29562577661432, + "bugs": 0.06033688972 } "# ); @@ -3003,36 +3007,38 @@ mod tests { |metric| { // The dekobon grammar consolidates the 8 primitive // type names (`byte`, `short`, `int`, `long`, `char`, - // `float`, `double`, `boolean`) under `type_identifier` - // — so they count as operands, not as distinct - // operators. Likewise numeric literals collapse to one - // `NumberLiteral` shape (no Hex/Octal/Binary/Decimal - // split), and `'x'` parses as `StringLiteral` (Groovy - // single-quoted strings) rather than as - // `CharacterLiteral`. Operators remaining in this - // fixture: `=`, the `class` keyword (#1552) and the - // class-body braces (only `{` is in the operator set). - // True/false collapse under one `BooleanLiteral`. - assert_eq!(metric.halstead.unique_operators(), 3); - assert_eq!(metric.halstead.unique_operands(), 27); + // `float`, `double`, `boolean`) under `type_identifier`; + // since #1568 each bills as a distinct operator keyed by + // its text, as Java's `java_primitive_ops` twin does. + // Numeric literals collapse to one `NumberLiteral` shape + // (no Hex/Octal/Binary/Decimal split), and `'x'` parses + // as `StringLiteral` (Groovy single-quoted strings) + // rather than as `CharacterLiteral`. + // expected: operators `=` x9, `class`, `{}` and the eight + // primitives (`boolean` twice) — n1 11, N1 20; operands + // `Prims`, the nine names and nine values — n2 19, N2 19. + assert_eq!(metric.halstead.unique_operators(), 11); + assert_eq!(metric.halstead.total_operators(), 20); + assert_eq!(metric.halstead.unique_operands(), 19); + assert_eq!(metric.halstead.total_operands(), 19); insta::assert_json_snapshot!( metric.halstead, @r#" { - "unique_operators": 3, - "total_operators": 11, - "unique_operands": 27, - "total_operands": 28, + "unique_operators": 11, + "total_operators": 20, + "unique_operands": 19, + "total_operands": 19, "length": 39, - "estimated_program_length": 133.1368500605771, - "purity_ratio": 3.413765386168644, + "estimated_program_length": 118.76437056043838, + "purity_ratio": 3.045240270780471, "vocabulary": 30, "volume": 191.36873322873222, - "difficulty": 1.5555555555555556, - "level": 0.6428571428571428, - "effort": 297.6846961335835, - "time": 16.53803867408797, - "bugs": 0.014861058740943886 + "difficulty": 5.5, + "level": 0.18181818181818182, + "effort": 1052.5280327580272, + "time": 58.47377959766818, + "bugs": 0.0344906352248725 } "# ); @@ -3117,10 +3123,13 @@ mod tests { #[cfg(feature = "groovy")] #[test] fn groovy_closure_operators_and_operands() { - check_metrics::("def double = { x -> x * 2 }", "foo.groovy", |metric| { + // `twice`, not the original `double`: Groovy reserves the + // primitive type names, so `def double` is not valid Groovy and + // `double` would bill as the primitive operator (#1568). + check_metrics::("def twice = { x -> x * 2 }", "foo.groovy", |metric| { // Closure with arrow-style parameter list. // Distinct operators: def, =, {}, ->, * = 5. - // Distinct operands: double, x, 2 = 3. + // Distinct operands: twice, x, 2 = 3. assert_eq!(metric.halstead.unique_operators(), 5); assert_eq!(metric.halstead.unique_operands(), 3); }); @@ -3698,6 +3707,70 @@ mod tests { ); } + /// #1573: an interpolation's alignment `,` and format `:` delimit the + /// field, as Python's format-spec `:` does (#1486), and bill nothing. + /// The alignment value is an ordinary expression, so `8`, `Width` + /// and `-w` keep their vocabulary. The guard reads the token's + /// parent: the last seven rows put the same `,` / `:` inside and + /// outside a hole in a position that is not a clause, and each keeps + /// billing. Pre-fix, every row with a clause billed one extra `,` + /// and/or `:`. + /// + /// Each row is `(label, source, operators, operands)`, the two + /// vocabularies space-separated. No row repeats a token, so the + /// expected `[n1, N1, n2, N2]` is the two vocabulary sizes, each twice. + #[cfg(feature = "csharp")] + #[test] + fn csharp_interpolation_clause_delimiters_are_not_operators_1573() { + const ROWS: &[(&str, &str, &str, &str)] = &[ + ("control", "s = $\"{x}\";", "; =", "s x"), + ("format", "s = $\"{x:F2}\";", "; =", "s x"), + ("alignment", "s = $\"{x,8}\";", "; =", "8 s x"), + ("both", "s = $\"{x,8:F2}\";", "; =", "8 s x"), + // a `const` width is a `constant_expression` the spec allows + ("named width", "s = $\"{x,Width}\";", "; =", "Width s x"), + ("negated width", "s = $\"{x,-w}\";", "- ; =", "s w x"), + // the format string's `,` is text, not a token + ("comma in format", "s = $\"{x:#,##0}\";", "; =", "s x"), + ("raw", "s = $$\"\"\"{{x,8:F2}}\"\"\";", "; =", "8 s x"), + ("verbatim", "s = $@\"{x:F2}\";", "; =", "s x"), + ("call in hole", "s = $\"{f(a, b)}\";", "() , ; =", "a b f s"), + ( + "ternary in hole", + "s = $\"{(a ? b : c)}\";", + "() : ; = ?", + "a b c s", + ), + ( + "named arg in hole", + "s = $\"{f(x: 1)}\";", + "() : ; =", + "1 f s x", + ), + ("ternary", "s = a ? b : c;", ": ; = ?", "a b c s"), + ("named argument", "f(x: 1);", "() : ;", "1 f x"), + ( + "case label", + "switch (x) { case 1: break; }", + "() : ; break case switch {}", + "1 x", + ), + ("argument list", "f(a, b);", "() , ;", "a b f"), + ]; + for (label, source, operators, operands) in ROWS { + // `bca ops` sorts its vocabulary, so the observation is + // compared as returned; the rows are written in that order. + let operators: Vec<&str> = operators.split(' ').collect(); + let operands: Vec<&str> = operands.split(' ').collect(); + let ops = ops_of::(source, "foo.cs"); + assert_eq!(ops.operators, operators, "{label}: {source:?}"); + assert_eq!(ops.operands, operands, "{label}: {source:?}"); + let n1 = operators.len() as u64; + let n2 = operands.len() as u64; + assert_halstead_counts::(source, "foo.cs", [n1, n1, n2, n2], label); + } + } + // C# half of #1380 — see `java_self_and_super_references_are_operands` // for the structural argument. `base` moves with `this`: both are // receivers of a member access. @@ -6197,8 +6270,8 @@ end", } /// #1552: Java's type-declaration keywords are operators, as C#'s - /// are. The class-literal row pins the guard: `C.class` is one - /// operand, and its `class` leaf bills nothing of its own. + /// are. The class-literal row pins the guard: the `class` tail of + /// `C.class` is an operand (#1566), not a declaration keyword. #[cfg(feature = "java")] #[test] fn java_declaration_keywords_are_operators_1552() { @@ -6214,7 +6287,7 @@ end", // `@interface`, `{}` ("annotation type", "@interface N {}\n", [2, 2]), // `class`, `{}`, `=`, `.`, `;` — one `class`: the literal's - // is part of the `C.class` operand + // tail is an operand (#1566) ("class literal", "class U { Object o = C.class; }\n", [5, 5]), ]; assert_operator_rows::("Foo.java", ROWS); @@ -6223,9 +6296,9 @@ end", /// #1560: a Java type name is an operand, as in C#, Rust, the C /// family, Go, Groovy and TypeScript. Each row isolates one position /// of the single `type_identifier` kind. The class-literal rows pin - /// that a type inside `Foo.class` is not billed again on top of the - /// literal's own operand, whether it sits there directly or under an - /// array or scoped type; the `var` rows, that the inferred-type + /// that a type inside `Foo.class` bills like any other (#1566), + /// whether it sits there directly or under an array or scoped type, + /// beside the literal's `class` tail; the `var` rows, that the inferred-type /// keyword bills nothing while a variable named `var` stays an /// operand. #[cfg(feature = "java")] @@ -6252,23 +6325,23 @@ end", ), // `C`, `m`, `Ex` ("throws", "class C { void m() throws Ex {} }\n", [3, 3]), - // `C`, `Object`, `a`, `Foo.class` + // `C`, `Object`, `a`, `Foo`, `class` ( "class literal", "class C { Object a = Foo.class; }\n", - [4, 4], + [5, 5], ), - // `C`, `Object`, `a`, `Foo[].class` + // `C`, `Object`, `a`, `Foo`, `class` ( "array class literal", "class C { Object a = Foo[].class; }\n", - [4, 4], + [5, 5], ), - // `C`, `Object`, `a`, `java.util.List.class` + // `C`, `Object`, `a`, `java`, `util`, `List`, `class` ( "scoped class literal", "class C { Object a = java.util.List.class; }\n", - [4, 4], + [7, 7], ), // `C`, `m`, `x`, `1` — no `var` ( @@ -6286,6 +6359,217 @@ end", assert_operand_rows::("Foo.java", ROWS); } + /// One class literal's own vocabulary: the source spelling, then the + /// operators and operands it adds to the fixture's frame. + #[cfg(any(feature = "groovy", feature = "java"))] + type ClassLiteralParts = ( + &'static str, + &'static [&'static str], + &'static [&'static str], + ); + + /// #1566: the class-literal shapes Java and Groovy spell alike. The + /// expected parts are what Groovy — untouched by #1566 — already + /// billed for `Foo.class`: the type name an operand, the `.` and + /// `[]` operators, and the `class` tail an operand. A primitive or + /// `void` receiver is an operator keyed by its spelling, as Java's + /// keyword is; Groovy joined that half in #1568. Both tests below + /// read this one table, so the two languages cannot drift apart. + #[cfg(any(feature = "groovy", feature = "java"))] + const SHARED_CLASS_LITERALS: &[ClassLiteralParts] = &[ + ("Foo.class", &["."], &["Foo", "class"]), + ("Foo[].class", &[".", "[]"], &["Foo", "class"]), + ( + "java.util.List.class", + &["."], + &["List", "class", "java", "util"], + ), + ("int.class", &[".", "int"], &["class"]), + ("int[].class", &[".", "[]", "int"], &["class"]), + ("void.class", &[".", "void"], &["class"]), + ]; + + /// Asserts each literal's operator and operand vocabularies inside + /// `frame`, which spells the fixture around it and contributes + /// `frame_ops` / `frame_operands` of its own. + #[cfg(any(feature = "groovy", feature = "java"))] + #[track_caller] + fn assert_class_literal_parts( + file: &str, + frame: fn(&str) -> String, + frame_ops: &[&str], + frame_operands: &[&str], + rows: &[ClassLiteralParts], + ) { + let merged = |frame: &[&str], parts: &[&str]| { + let mut all: Vec = frame.iter().chain(parts).map(|s| (*s).to_owned()).collect(); + all.sort_unstable(); + all.dedup(); + all + }; + for (literal, ops, operands) in rows { + let got = ops_of::(&frame(literal), file); + assert_eq!( + got.operators, + merged(frame_ops, ops), + "operators of {literal}" + ); + assert_eq!( + got.operands, + merged(frame_operands, operands), + "operands of {literal}" + ); + } + } + + /// #1566: a Java class literal bills its parts. JLS §15.8.2 makes it + /// a primary expression rather than a literal (§3.10 lists none), so + /// the type name is an operand as anywhere else, `.` and `[]` are + /// operators, and the `class` tail is the operand they act on — the + /// split Kotlin's and PHP's `Foo::class` and Groovy's `Foo.class` + /// already made. Before the fix the literal billed as one operand + /// spelled by its whole text while `.`, `[]` and a primitive inside + /// it also billed as operators. The shared rows take their expected + /// values from Groovy, whose primitive and `void` rows joined them + /// in #1568. + #[cfg(feature = "java")] + #[test] + fn java_class_literal_bills_its_parts_1566() { + const FRAME_OPS: &[&str] = &[";", "=", "class", "{}"]; + const FRAME_OPERANDS: &[&str] = &["C", "Object", "c"]; + let frame = |literal: &str| format!("class C {{ Object c = {literal}; }}\n"); + assert_class_literal_parts::( + "Foo.java", + frame, + FRAME_OPS, + FRAME_OPERANDS, + SHARED_CLASS_LITERALS, + ); + // The totals pin the multiplicities a vocabulary cannot: the frame + // bills four operators (`class`, `{}`, `=`, `;`) and three operands + // (`C`, `Object`, `c`), and each literal adds its parts once — + // `java.util.List.class` its three `.` and four names. + assert_halstead_rows::( + "Foo.java", + &[ + ("int", "class C { Object c = int.class; }\n", [6, 4]), + ("array", "class C { Object c = Foo[].class; }\n", [6, 5]), + ( + "scoped", + "class C { Object c = java.util.List.class; }\n", + [7, 7], + ), + ("void", "class C { Object c = void.class; }\n", [6, 4]), + ], + |h| [h.total_operators(), h.total_operands()], + ); + } + + /// #1566's oracle: Groovy bills the shared class-literal shapes the + /// way Java now does. A Groovy field needs no `;`. + #[cfg(feature = "groovy")] + #[test] + fn groovy_class_literal_bills_its_parts_1566() { + assert_class_literal_parts::( + "foo.groovy", + |literal| format!("class C {{ Object c = {literal} }}\n"), + &["=", "class", "{}"], + &["C", "Object", "c"], + SHARED_CLASS_LITERALS, + ); + } + + /// #1568: a Groovy primitive type bills as the operator Java's + /// keyword does, keyed by its spelling, in every position the + /// grammar gives it: a `type_identifier` where a type is declared, + /// a plain `identifier` where the grammar misreads an uninitialised + /// local (`int t` parses as two expression statements). Each row's + /// operators are what the Java twin of the issue fixture bills, which + /// `groovy_primitive_type_matches_java_1568` checks directly. + #[cfg(feature = "groovy")] + #[test] + fn groovy_primitive_types_are_operators_1568() { + const ROWS: &[(&str, &[&str], &[&str])] = &[ + ( + "class A { boolean m() { int x = 1; } }\n", + &["()", ";", "=", "boolean", "class", "int", "{}"], + &["1", "A", "m", "x"], + ), + ( + "void m(long a, char b) {}\n", + &["()", ",", "char", "long", "void", "{}"], + &["a", "b", "m"], + ), + ( + "def m() { short s\n double d }\n", + &["()", "def", "double", "short", "{}"], + &["d", "m", "s"], + ), + ( + "byte[] a = (float) b as byte\n", + &["()", "=", "[]", "as", "byte", "float"], + &["a", "b"], + ), + // A user type's name stays an operand, however close its + // spelling: only the exact keyword bills. + ( + "Integer a\nInt b\ninte c\n", + &[], + &["Int", "Integer", "a", "b", "c", "inte"], + ), + ]; + for (source, operators, operands) in ROWS { + let ops = ops_of::(source, "foo.groovy"); + assert_eq!(ops.operators, *operators, "operators of {source:?}"); + assert_eq!(ops.operands, *operands, "operands of {source:?}"); + } + } + + /// #1568's oracle: the issue's fixture scores the same in Groovy and + /// Java, vocabulary and multiplicities alike. + #[cfg(all(feature = "groovy", feature = "java"))] + #[test] + fn groovy_primitive_type_matches_java_1568() { + let source = "class A { boolean m() { int x = 1; } }\n"; + let groovy = ops_of::(source, "foo.groovy"); + let java = ops_of::(source, "Foo.java"); + assert_eq!(groovy.operators, java.operators); + assert_eq!(groovy.operands, java.operands); + // `()`, `;`, `=`, `boolean`, `class`, `int`, `{}` x2 / `A`, `m`, + // `x`, `1` + assert_halstead_rows::( + "foo.groovy", + &[("issue fixture", source, [8, 4])], + |h| [h.total_operators(), h.total_operands()], + ); + assert_halstead_rows::("Foo.java", &[("issue fixture", source, [8, 4])], |h| { + [h.total_operators(), h.total_operands()] + }); + } + + /// #1568: Groovy reserves the primitive type names, so one can name + /// something only as a member selector after a `.`-family operator + /// or as a map or named-argument label. Each position stays an + /// operand; each row is its own source so a selector kind dropped + /// from the gate fails by name. + #[cfg(feature = "groovy")] + #[test] + fn groovy_primitive_spelled_names_stay_operands_1568() { + for source in [ + "x.int\n", + "x?.int\n", + "x??.int\n", + "x*.int\n", + "x.@int\n", + "x.&int\n", + "x::int\n", + "def m = [int: 1]\n", + "f(int: 1)\n", + ] { + assert_keywords_are_operands_only::(source, "foo.groovy", &["int"]); + } + } + /// #1552: Groovy's type-declaration keywords are operators, as /// Java's are. `String.class` is a `field_access` whose `class` is a /// plain identifier, so it stays an operand with no guard needed. @@ -6301,8 +6585,8 @@ end", ("trait", "trait T {}\n", [2, 2]), // `enum`, `{}` ("enum", "enum E { A }\n", [2, 2]), - // `record`, `()`, `{}` - ("record", "record R(int x) {}\n", [3, 3]), + // `record`, `()`, `int`, `{}` — as Java's row, since #1568 + ("record", "record R(int x) {}\n", [4, 4]), // `def`, `=`, `.` — no `class` ("class reference", "def a = String.class\n", [3, 3]), // `=` twice: a script-level `record` / `trait` used as a name @@ -6313,6 +6597,256 @@ end", assert_operator_rows::("foo.groovy", ROWS); } + /// #1565: Java bills every modifier, as C# and Kotlin do, each as + /// its keyword leaf and never as the `modifiers` wrapper. Before the + /// fix only `abstract`, `final`, `transient`, `synchronized` and + /// `default` billed. The rows from "import static" on are controls: + /// a directive's `static` and the contextual words used as names add + /// no operator. + #[cfg(feature = "java")] + #[test] + fn java_modifiers_are_operators_1565() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}`, `public`, `private`, `protected`, `int` x3, + // `;` x3 + ( + "visibility", + "class C { public int a; private int b; protected int c; }\n", + [7, 11], + ), + // `class`, `{}`, `static`, `final`, `int` x2, `=`, `;` x2, + // `transient`, `volatile` + ( + "field modifiers", + "class C { static final int a = 1; transient volatile int b; }\n", + [9, 11], + ), + // `class`, `{}` x3, `native`, `synchronized`, `strictfp`, + // `void` x3, `()` x3, `;` + ( + "method modifiers", + "class C { native void n(); synchronized void s() {} strictfp void f() {} }\n", + [8, 14], + ), + // `sealed`, `class` x2, `permits`, `{}` x2, `non-sealed`, + // `extends` + ( + "sealed hierarchy", + "sealed class S permits T {}\nnon-sealed class T extends S {}\n", + [6, 8], + ), + // `class`, `{}` x2, `static`: the initializer's `static` + ("static initializer", "class C { static {} }\n", [3, 4]), + // `class`, `{}` x2, `public`, `()`, `return`, `;`: the + // annotation beside the modifier bills nothing of its own + ( + "annotation in modifiers", + "class C { @Override public String toString() { return s; } }\n", + [6, 7], + ), + // `.` x3, `;`, `class`, `{}`: no `static` + ( + "import static", + "import static java.lang.Math.max;\nclass C {}\n", + [4, 6], + ), + // `{}`, `;`: no `static`, and `transitive` stays unbilled + // with the rest of the module directive + ( + "requires static", + "module m { requires static transitive a; }\n", + [2, 2], + ), + // `class`, `{}` x2, `void`, `()`, `=` x3, `;` x3, `int` + ( + "names", + "class C { void m() { var sealed = 1; int permits = 2; sealed = permits; } }\n", + [7, 12], + ), + // `class`, `{}`, `int` x2, `,`, `;` x2, `=`, `-` + ( + "non-sealed as a subtraction", + "class C { int non, sealed; int z = non-sealed; }\n", + [7, 9], + ), + ]; + assert_operator_rows::("Foo.java", ROWS); + } + + /// #1565: the issue's fixture, by vocabulary. Each modifier is + /// listed by its own word, never by `modifiers`, and the contextual + /// words used as names are operands only. No getter edit makes that + /// second half fail, since Java's arms carry no guard on those + /// words: it pins the grammar emitting an `identifier` for a name, + /// so a bump that routes one to the keyword kind trips it. + #[cfg(feature = "java")] + #[test] + fn java_modifier_vocabulary_1565() { + let ops = ops_of::( + "public abstract class C { private static final int a = 1; \ + protected transient int b; public synchronized void m() {} }\n", + "Foo.java", + ); + assert_eq!( + ops.operators, + [ + "()", + ";", + "=", + "abstract", + "class", + "final", + "int", + "private", + "protected", + "public", + "static", + "synchronized", + "transient", + "void", + "{}", + ] + ); + assert_eq!(ops.operands, ["1", "C", "a", "b", "m"]); + + assert_keywords_are_operands_only::( + "class C { int sealed() { return permits; } void m() { sealed(); } }\n", + "Foo.java", + &["sealed", "permits"], + ); + } + + /// #1565: Groovy bills every modifier, as Java now does. Its grammar + /// has no `modifiers` wrapper, so each leaf sits directly under its + /// declaration. The rows declare with `def`, so the only primitive + /// type counted is `void`, an operator since #1568. The last two + /// rows are controls. + #[cfg(feature = "groovy")] + #[test] + fn groovy_modifiers_are_operators_1565() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}`, `public`, `private`, `protected`, `def` x3 + ( + "visibility", + "class C {\n public def a\n private def b\n protected def c\n}\n", + [6, 8], + ), + // `class`, `{}`, `static`, `final`, `def` x2, `=`, + // `transient`, `volatile` + ( + "field modifiers", + "class C {\n static final def a = 1\n transient volatile def b\n}\n", + [8, 9], + ), + // `class`, `{}` x3, `native`, `synchronized`, `strictfp`, + // `void` x3 (#1568), `()` x3 + ( + "method modifiers", + "class C {\n native void n()\n synchronized void s() {}\n \ + strictfp void f() {}\n}\n", + [7, 13], + ), + // `sealed`, `class` x2, `permits`, `{}` x2, `non-sealed`, + // `extends` + ( + "sealed hierarchy", + "sealed class S permits T {}\nnon-sealed class T extends S {}\n", + [6, 8], + ), + // `sealed` x2, `interface`, `trait`, `permits` x2, `{}` x2 + ( + "sealed interface and trait", + "sealed interface I permits A {}\nsealed trait T permits A {}\n", + [5, 8], + ), + // `class`, `{}` x3, `static` x2, `void` (#1568), `()`: an + // initializer and a script-level method + ( + "static initializer and method", + "class C {\n static {}\n}\nstatic void top() {}\n", + [5, 8], + ), + // `.` x3: no `static`, as `import` bills nothing + ( + "import static", + "import static java.lang.Math.max\n", + [1, 3], + ), + // `=` x2, `.`, `()` x2, `def`: a statement opening with + // `sealed` as a name fails into an `ERROR` holding the + // keyword leaf, which bills nothing + ( + "contextual names", + "sealed = 1\nsealed.call()\ndef permits = 2\nfoo(permits)\n", + [4, 6], + ), + ]; + assert_operator_rows::("foo.groovy", ROWS); + } + + /// #1565: the issue's Groovy fixture, by vocabulary, plus the + /// modifier words in positions the grammar parses as a name. Unlike + /// Java, Groovy accepts even its hard keywords as a map key, a named + /// argument and a member name. No getter edit makes that half fail: + /// it pins the grammar emitting an `identifier` there, so a bump + /// that routes one to the keyword kind trips it. + #[cfg(feature = "groovy")] + #[test] + fn groovy_modifier_vocabulary_1565() { + let ops = ops_of::( + "public abstract class C { private static final def a = 1; protected def m() {} }\n", + "foo.groovy", + ); + assert_eq!( + ops.operators, + [ + "()", + ";", + "=", + "abstract", + "class", + "def", + "final", + "private", + "protected", + "public", + "static", + "{}", + ] + ); + assert_eq!(ops.operands, ["1", "C", "a", "m"]); + + assert_keywords_are_operands_only::( + "x = sealed\nfoo(permits)\ndef m = [static: 1, public: 2]\n\ + foo(native: 1, volatile: 2)\ny = obj.private\n", + "foo.groovy", + &[ + "sealed", "permits", "static", "public", "native", "volatile", "private", + ], + ); + } + + /// #1565's oracle: Groovy accepts Java's modifier-heavy fixture + /// verbatim, and the two languages bill it alike, vocabulary and + /// multiplicities. Neither side's expectation is typed by the fix + /// alone: a modifier one getter drops shows as a disagreement. + #[cfg(all(feature = "groovy", feature = "java"))] + #[test] + fn groovy_modifiers_match_java_1565() { + let source = "public abstract class C { private static final int a = 1; \ + protected transient int b; public synchronized void m() {} }\n"; + let groovy = ops_of::(source, "foo.groovy"); + let java = ops_of::(source, "Foo.java"); + assert_eq!(groovy.operators, java.operators); + assert_eq!(groovy.operands, java.operands); + // `public` x2, `abstract`, `class`, `{}` x2, `private`, `static`, + // `final`, `int` x2, `=`, `;` x2, `protected`, `transient`, + // `synchronized`, `void`, `()` + let rows = [("modifier fixture", source, [15, 19])]; + assert_operator_rows::("foo.groovy", &rows); + assert_operator_rows::("Foo.java", &rows); + } + /// #1552 rows shared by the two C++ clones. The template-parameter /// rows pin the synonym: `class T` is `typename T`, and the two /// spellings score alike. Both were unbilled until #1557 billed @@ -6386,6 +6920,280 @@ end", assert_operator_rows::("foo.c", ROWS); } + /// #1578: C bills each storage class, function specifier and type + /// qualifier as its keyword leaf, never as the + /// `storage_class_specifier` / `type_qualifier` wrapper. Before the + /// fix none of them billed. + #[cfg(feature = "c")] + #[test] + fn c_modifiers_are_operators_1578() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `static`, `extern`, `register`, `auto`, `int` x4, `;` x4 + ( + "storage classes", + "static int a;\nextern int b;\nregister int c;\nauto int d;\n", + [6, 12], + ), + // `const`, `volatile`, `int` x3, `;` x3, `*`, `restrict`, + // `_Atomic` + ( + "qualifiers", + "const volatile int a;\nint * restrict p;\n_Atomic int g;\n", + [7, 11], + ), + // `inline`, `int`, `()` x2, `void` x3, `;` x2, `_Noreturn` + ( + "function specifiers", + "inline int f(void);\n_Noreturn void g(void);\n", + [6, 10], + ), + // `__inline`, `int` x3, `()`, `void`, `;` x3, `__thread`, `*`, + // `__restrict__`: each vendor spelling is its own operator + ( + "vendor spellings", + "__inline int f(void);\n__thread int t;\nint * __restrict__ q;\n", + [8, 12], + ), + // `int`, `*`, `const`, `=`, `;`: a pointer's own qualifier + ("pointer qualifier", "int * const p = 0;\n", [5, 5]), + // `_Alignas`, `()`, `int`, `;` + ("alignas", "_Alignas(8) int m;\n", [4, 4]), + // `int`, `=`, `__extension__`, `;`: the expression prefix + ("extension expression", "int q = __extension__ 1;\n", [4, 4]), + ]; + assert_operator_rows::("foo.c", ROWS); + } + + /// #1578: the issue's C fixture, by vocabulary. + #[cfg(feature = "c")] + #[test] + fn c_modifier_vocabulary_1578() { + let ops = ops_of::( + "static inline const volatile int a = 1;\nextern int b;\nregister int c;\n", + "foo.c", + ); + assert_eq!( + ops.operators, + [ + ";", "=", "const", "extern", "inline", "int", "register", "static", "volatile", + ] + ); + assert_eq!(ops.operands, ["1", "a", "b", "c"]); + } + + /// #1578 rows shared by the two C++ clones: C's modifiers plus C++'s + /// own. Each bills as its keyword leaf, never as the + /// `access_specifier` / `virtual_specifier` / `type_qualifier` / + /// `explicit_function_specifier` wrapper. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const CPP_MODIFIER_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}`, `public`, `protected`, `private`, `:` x3, + // `int` x3, `;` x4 + ( + "access labels", + "class C { public: int a; protected: int b; private: int c; };\n", + [8, 15], + ), + // `struct`, `{}`, `static`, `int` x2, `;` x4, `mutable`, + // `virtual`, `void`, `()`, `const`, `override`, `final` + ( + "member modifiers", + "struct S { static int a; mutable int b; virtual void f() const override final; };\n", + [12, 16], + ), + // `struct`, `{}`, `friend`, `class`, `;` x3, `explicit`, `()`, + // `int` + ( + "friend and explicit", + "struct S { friend class D; explicit S(int); };\n", + [8, 10], + ), + // `extern`, `inline`, `constexpr`, `int`, `()`, `{}`, `return`, + // `;`: the `"C"` is an operand + ( + "linkage specification", + "extern \"C\" inline constexpr int g() { return 1; }\n", + [8, 8], + ), + // `consteval`, `int` x2, `()`, `;` x2, `constinit`, + // `thread_local`, `=` + ( + "C++20 specifiers", + "consteval int h();\nconstinit thread_local int t = 1;\n", + [7, 9], + ), + // `void` x2, `()` x3, `noexcept` x2, `;` x2: the leaf bills, the + // `noexcept` wrapper does not + ( + "exception specification", + "void k() noexcept;\nvoid k2() noexcept(true);\n", + [4, 9], + ), + // `struct`, `final`, `{}` x3, `;` x3, `class` x2, `:` x2, + // `public`, `private`, `virtual` + ( + "base clauses", + "struct B final {};\nclass E : public B {};\nclass F : private virtual B {};\n", + [9, 15], + ), + // `void`, `()` x2, `{}` x2, `if`, `constexpr` + ( + "if constexpr", + "void f() { if constexpr (true) {} }\n", + [5, 7], + ), + // `inline`, `namespace`, `{}` + ("inline namespace", "inline namespace v1 {}\n", [3, 3]), + // `int` x2, `=` x2, `;` x2: the contextual words as names add + // no operator + ( + "contextual names", + "int override = 1;\nint final = override;\n", + [3, 6], + ), + ]; + + /// One call per clone, so reverting either arm fails its own row; + /// Mozcpp owns no file extension, so this is its only lib coverage. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_modifiers_are_operators_1578() { + assert_operator_rows::("foo.cpp", CPP_MODIFIER_ROWS); + assert_operator_rows::("foo.cpp", CPP_MODIFIER_ROWS); + } + + /// #1578: the issue's C++ fixture, by vocabulary, in both clones, + /// plus `override` and `final` used as names. No getter edit makes + /// that second half fail, since the arms carry no guard on those + /// words: it pins the grammar emitting an `identifier` for a name, + /// so a bump that routes one to the keyword kind trips it. + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_modifier_vocabulary_1578() { + const SOURCE: &str = "class C { public: static int a; \ + virtual void f() const override final; mutable int b; };\n\ + extern \"C\" inline constexpr int g() { return 1; }\n"; + const NAMES: &str = "int override = 1;\nint final() { return override; }\n"; + let expected_operators = [ + "()", + ":", + ";", + "class", + "const", + "constexpr", + "extern", + "final", + "inline", + "int", + "mutable", + "override", + "public", + "return", + "static", + "virtual", + "void", + "{}", + ]; + let expected_operands = ["\"C\"", "1", "C", "a", "b", "f", "g"]; + + let cpp = ops_of::(SOURCE, "foo.cpp"); + assert_eq!(cpp.operators, expected_operators); + assert_eq!(cpp.operands, expected_operands); + let mozcpp = ops_of::(SOURCE, "foo.cpp"); + assert_eq!(mozcpp.operators, expected_operators); + assert_eq!(mozcpp.operands, expected_operands); + + assert_keywords_are_operands_only::(NAMES, "foo.cpp", &["override", "final"]); + assert_keywords_are_operands_only::(NAMES, "foo.cpp", &["override", "final"]); + } + + /// #1578: every modifier spelling the arms list bills under its own + /// name, one fixture line apiece, so dropping any one kind from an + /// arm fails its own entry. The row tests above pin the counts; this + /// one pins the spellings they leave out. + #[cfg(any(feature = "c", feature = "cpp", feature = "mozcpp", feature = "objc"))] + #[track_caller] + fn assert_modifier_spellings_bill(file: &str, rows: &[(&str, &str)]) { + for (spelling, source) in rows { + let ops = ops_of::(source, file); + assert!( + ops.operators.iter().any(|o| o == spelling), + "{file}: `{spelling}` must be an operator in {source:?}; operators were {:?}", + ops.operators + ); + assert!( + !ops.operands.iter().any(|o| o == spelling), + "{file}: `{spelling}` must not be an operand; operands were {:?}", + ops.operands + ); + } + } + + /// The spellings C and C++ share, each in a declaration both + /// grammars parse without error. + #[cfg(any(feature = "c", feature = "cpp", feature = "mozcpp", feature = "objc"))] + const C_FAMILY_MODIFIER_SPELLINGS: &[(&str, &str)] = &[ + ("static", "static int a;\n"), + ("extern", "extern int a;\n"), + ("register", "register int a;\n"), + ("inline", "inline int f(void);\n"), + ("__inline", "__inline int f(void);\n"), + ("__inline__", "__inline__ int f(void);\n"), + ("__forceinline", "__forceinline int f(void);\n"), + ("thread_local", "thread_local int a;\n"), + ("__thread", "__thread int a;\n"), + ("const", "const int a = 1;\n"), + ("constexpr", "constexpr int a = 1;\n"), + ("volatile", "volatile int a;\n"), + ("restrict", "int * restrict p;\n"), + ("__restrict__", "int * __restrict__ p;\n"), + ("_Atomic", "_Atomic int a;\n"), + ("_Noreturn", "_Noreturn void f(void);\n"), + ("noreturn", "noreturn void f(void);\n"), + ("_Nonnull", "int * _Nonnull p;\n"), + ("alignas", "alignas(8) int a;\n"), + ("_Alignas", "_Alignas(8) int a;\n"), + ("__extension__", "__extension__ int a;\n"), + // The one place the grammar emits `__volatile__`: as a GNU `asm` + // qualifier, where the plain spelling bills too. + ( + "__volatile__", + "void f(void) { __asm__ __volatile__(\"nop\"); }\n", + ), + ("volatile", "void f(void) { asm volatile(\"nop\"); }\n"), + ]; + + #[cfg(feature = "c")] + #[test] + fn c_modifier_spellings_bill_1578() { + assert_modifier_spellings_bill::("foo.c", C_FAMILY_MODIFIER_SPELLINGS); + // C's storage class `auto`; C++ has none (see the Cpp arm). + assert_modifier_spellings_bill::("foo.c", &[("auto", "auto int a;\n")]); + } + + #[cfg(all(feature = "cpp", feature = "mozcpp"))] + #[test] + fn cpp_modifier_spellings_bill_1578() { + const CPP_ONLY: &[(&str, &str)] = &[ + ("virtual", "struct S { virtual void f(); };\n"), + ("mutable", "struct S { mutable int a; };\n"), + ("constinit", "constinit int a = 1;\n"), + ("consteval", "consteval int f() { return 1; }\n"), + ("explicit", "struct S { explicit S(int); };\n"), + ("friend", "struct S { friend class D; };\n"), + ("public", "class C { public: int a; };\n"), + ("private", "class C { private: int a; };\n"), + ("protected", "class C { protected: int a; };\n"), + ("override", "struct S { void f() override; };\n"), + ("final", "struct S final {};\n"), + ("noexcept", "void f() noexcept;\n"), + ]; + for rows in [C_FAMILY_MODIFIER_SPELLINGS, CPP_ONLY] { + assert_modifier_spellings_bill::("foo.cpp", rows); + assert_modifier_spellings_bill::("foo.cpp", rows); + } + } + /// #1552: ObjC bills C's `struct` and its own class and protocol /// declaration keywords. `@class` already billed its `@` and stays /// as it was. @@ -6407,6 +7215,203 @@ end", assert_operator_rows::("foo.m", ROWS); } + /// #1579: ObjC bills C's declaration modifiers as `c.rs` does + /// (#1578), and its own: ivar visibility labels, property attribute + /// words, ARC ownership and nullability qualifiers, protocol section + /// labels, method-type qualifiers and generic variance. Before the + /// fix none of them billed. + #[cfg(feature = "objc")] + #[test] + fn objc_modifiers_are_operators_1579() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `static`, `const`, `int`, `=`, `;`: the issue's first line + ("C modifiers", "static const int a = 1;\n", [5, 5]), + // `@interface`, `{}`, `@public`, `@private`, `int` x2, `;` x2 + ( + "ivar visibility", + "@interface C { @public int x; @private int y; }\n@end\n", + [6, 8], + ), + // `@interface`, `()`, `nonatomic`, `,` x2, `strong`, + // `getter`, `=`, `;`: `isY` is an operand, `id` and + // `@property` bill nothing + ( + "property attributes", + "@interface C\n@property (nonatomic, strong, getter=isY) id y;\n@end\n", + [8, 9], + ), + // `__weak`, `__strong`, `__block`, `int`, `;` x3 + ( + "ownership qualifiers", + "__weak id a;\n__strong id b;\n__block int c;\n", + [5, 7], + ), + // `*` x2, `_Nullable`, `nullable`, `;` x2 + ( + "nullability qualifiers", + "NSString * _Nullable f;\nNSString * nullable k;\n", + [4, 6], + ), + // `@protocol`, `@optional`, `@required`, `-` x2, `()` x3, + // `void` x2, `;` x2, `oneway`, `:`, `out`, `*` + ( + "protocol sections and method qualifiers", + "@protocol P\n@optional\n- (void)m;\n@required\n\ + - (oneway void)n:(out id *)b;\n@end\n", + [11, 16], + ), + // `@interface`, `<>`, `__covariant`, `:` + ( + "generic variance", + "@interface G<__covariant T> : NSObject\n@end\n", + [4, 4], + ), + ]; + assert_operator_rows::("foo.m", ROWS); + } + + /// #1579: the `nullable` qualifier and the `nullable` property + /// attribute are one operator. The attribute has no keyword kind, so + /// it is keyed by text, and keying the qualifier by kind would list + /// the one word twice in `n1`. + #[cfg(feature = "objc")] + #[test] + fn objc_nullable_is_one_operator_1579() { + const SOURCE: &str = "@interface C\n@property (nullable) id y;\n- (nullable id)m;\n@end\n"; + // `@interface`, `()` x2, `nullable` x2, `-`, `;` x2 + assert_operator_rows::("foo.m", &[("nullable", SOURCE, [5, 8])]); + let ops = ops_of::(SOURCE, "foo.m"); + assert_eq!(ops.operators, ["()", "-", ";", "@interface", "nullable"]); + assert_eq!(ops.operands, ["C", "m", "y"]); + } + + /// #1579's oracle: a `.m` is a C translation unit, so C's + /// modifier-heavy declarations score alike in both grammars, + /// vocabulary and multiplicities. Neither expectation is typed by + /// the fix: a modifier one getter drops shows as a disagreement. + #[cfg(all(feature = "c", feature = "objc"))] + #[test] + fn objc_c_modifiers_match_c_1579() { + let source = "static inline const volatile int a = 1;\nextern int b;\n\ + register int c;\nauto int d;\nint * restrict p;\n_Atomic int g;\n\ + __thread int t;\n_Noreturn void h(void);\n_Alignas(8) int m;\n"; + let objc = ops_of::(source, "foo.m"); + let c = ops_of::(source, "foo.c"); + assert_eq!(objc.operators, c.operators); + assert_eq!(objc.operands, c.operands); + // `static`, `inline`, `const`, `volatile`, `int` x8, `=`, `;` x9, + // `extern`, `register`, `auto`, `*`, `restrict`, `_Atomic`, + // `__thread`, `_Noreturn`, `void` x2, `()` x2, `_Alignas` + let rows = [("C modifiers", source, [18, 35])]; + assert_operator_rows::("foo.m", &rows); + assert_operator_rows::("foo.c", &rows); + } + + /// #1579: every modifier word the ObjC arm lists bills under its own + /// name, one fixture apiece, so dropping any kind from the arm fails + /// its own entry. The spellings C shares with C++ run through the + /// #1578 table; `OBJC_ONLY` holds the rest, C's `auto` included, plus + /// a sample of the text-keyed property attributes. + #[cfg(feature = "objc")] + #[test] + fn objc_modifier_spellings_bill_1579() { + const OBJC_ONLY: &[(&str, &str)] = &[ + ("auto", "auto int a;\n"), + ("__const", "__const int a;\n"), + ("_Complex", "double _Complex z;\n"), + ("__complex", "double __complex z;\n"), + ("nullable", "NSString * nullable p;\n"), + ("_Nullable", "NSString * _Nullable p;\n"), + ("__nullable", "NSString * __nullable p;\n"), + ("_Nullable_result", "NSString * _Nullable_result p;\n"), + ("_Null_unspecified", "NSString * _Null_unspecified p;\n"), + ("__nonnull", "NSString * __nonnull p;\n"), + ("__autoreleasing", "__autoreleasing id a;\n"), + ("__block", "__block int a;\n"), + ("__strong", "__strong id a;\n"), + ("__unsafe_unretained", "__unsafe_unretained id a;\n"), + ("__weak", "__weak id a;\n"), + ("__bridge", "void f(void) { id x = (__bridge id)p; }\n"), + ( + "__bridge_retained", + "void f(void) { void *x = (__bridge_retained void *)p; }\n", + ), + ( + "__bridge_transfer", + "void f(void) { id x = (__bridge_transfer id)p; }\n", + ), + ("__kindof", "__kindof NSView *v;\n"), + ( + "__covariant", + "@interface G<__covariant T> : NSObject\n@end\n", + ), + ( + "__contravariant", + "@interface G<__contravariant T> : NSObject\n@end\n", + ), + ("in", "@protocol P\n- (void)m:(in id)a;\n@end\n"), + ("out", "@protocol P\n- (void)m:(out id *)a;\n@end\n"), + ("inout", "@protocol P\n- (void)m:(inout id *)a;\n@end\n"), + ("bycopy", "@protocol P\n- (void)m:(bycopy id)a;\n@end\n"), + ("byref", "@protocol P\n- (void)m:(byref id)a;\n@end\n"), + ("oneway", "@protocol P\n- (oneway void)m;\n@end\n"), + ("@public", "@interface C { @public int x; }\n@end\n"), + ("@private", "@interface C { @private int x; }\n@end\n"), + ("@protected", "@interface C { @protected int x; }\n@end\n"), + ("@package", "@interface C { @package int x; }\n@end\n"), + ("@optional", "@protocol P\n@optional\n- (void)m;\n@end\n"), + ("@required", "@protocol P\n@required\n- (void)m;\n@end\n"), + ("copy", "@interface C\n@property (copy) id y;\n@end\n"), + ( + "readonly", + "@interface C\n@property (readonly) id y;\n@end\n", + ), + ("class", "@interface C\n@property (class) id y;\n@end\n"), + ( + "setter", + "@interface C\n@property (setter=sY:) id y;\n@end\n", + ), + ]; + assert_modifier_spellings_bill::("foo.m", C_FAMILY_MODIFIER_SPELLINGS); + assert_modifier_spellings_bill::("foo.m", OBJC_ONLY); + } + + /// #1579: the plain-word modifiers stay operands where they name a + /// variable, and a property attribute's `getter=` value is the + /// selector it names, not a modifier. The arms carry no guard on + /// these words, so this pins the grammar: it emits an `identifier` + /// for each name, and only the first word of a `property_attribute` + /// bills. + #[cfg(feature = "objc")] + #[test] + fn objc_modifier_words_as_names_stay_operands_1579() { + assert_keywords_are_operands_only::( + "int out = 1;\nint copy = 2;\nint oneway = 3;\nint strong = out + copy;\n", + "foo.m", + &["out", "copy", "oneway", "strong"], + ); + assert_keywords_are_operands_only::( + "@interface C\n@property (getter=atomic, setter=weak:) id y;\n@end\n", + "foo.m", + &["atomic", "weak"], + ); + } + + /// #1579: the Apple SDK macros the grammar files as storage classes + /// and qualifiers bill nothing, as Mozcpp's `MOZ_*` macros do: each + /// annotates a declaration, and a `.c` parses the spelling as a name. + #[cfg(feature = "objc")] + #[test] + fn objc_sdk_macros_bill_nothing_1579() { + // `int` x4, `;` x4, `()`, `void`, `{}`, `return`: the macros add + // nothing, and the declared names and `1` are the operands + const SOURCE: &str = "NS_INLINE int f(void) { return 1; }\nFOUNDATION_EXPORT int g;\n\ + IBOutlet int h;\n__unused int u;\n"; + let ops = ops_of::(SOURCE, "foo.m"); + assert_eq!(ops.operators, ["()", ";", "int", "return", "void", "{}"]); + assert_eq!(ops.operands, ["1", "f", "g", "h", "u"]); + } + /// #1556: Kotlin's `interface` bills as its `class` and `object` /// do. The modifier rows score one above the bare keyword since /// #1558: `enum` in `enum class` is a `class_modifier` like `data` @@ -6587,6 +7592,104 @@ end", ); } + /// #1564: `typealias`, `init`, `where`, `by`, `get` and `set` bill + /// as Kotlin's other declaration keywords do, and as C# bills its + /// `get` / `set` / `init` / `where`. Each is the keyword leaf, never + /// its wrapper. The rows after "accessors" are controls: the + /// `@get:` / `@set:` use-site targets and the words used as names + /// add no operator. + #[cfg(feature = "kotlin")] + #[test] + fn kotlin_declaration_keywords_are_operators_1564() { + const ROWS: &[(&str, &str, [u64; 2])] = &[ + // `typealias`, `=` + ("typealias", "typealias A = Int\n", [2, 2]), + // `class`, `init`, `{}` x2 + ("init", "class C {\n init {}\n}\n", [3, 4]), + // `class`, `<>`, `where`, `:` + ("class where", "class C where T : Any\n", [4, 4]), + // `fun`, `<>`, `()`, `where`, `:`, `{}` + ("function where", "fun f() where T : Any {}\n", [6, 6]), + // `val`, `by`, `{}` + ("property delegate", "val d by lazy { 1 }\n", [3, 3]), + // `class`, `()`, `:` x2, `by` + ("class delegation", "class D(d: I) : I by d\n", [4, 5]), + // `var`, `:`, `=` x3, `get`, `()` x2, `set`, `{}` + ( + "accessors", + "var p: Int = 0\n get() = field\n set(v) { field = v }\n", + [7, 10], + ), + // `var`, `=`, `:` x2: the use-site targets bill nothing, as + // `@field:` does + ("use-site targets", "@get:A @set:B var q = 0\n", [3, 4]), + // `val`, `=` + ( + "names", + "val get = 1\nval set = 2\nval by = 3\nval where = 4\nval init = 5\n", + [2, 10], + ), + // `fun`, `()` x4, `:`, `{}`, `.` x2 + ( + "calls", + "fun g(x: X) {\n x.get(by)\n x.set(where)\n init(x)\n}\n", + [5, 9], + ), + // `fun`, `()` x2, `{}`, `val`, `=`: valid Kotlin that kotlin-ng + // misreads, leaving the call's `get` leaf in an `ERROR` + ( + "call after a local val", + "fun g() {\n val f = 6\n get(1)\n}\n", + [5, 6], + ), + ]; + assert_operator_rows::("foo.kt", ROWS); + } + + /// #1564: the six words are listed by their own spelling, and the + /// accessor body's `field` stays an operand like any variable. + #[cfg(feature = "kotlin")] + #[test] + fn kotlin_declaration_keyword_vocabulary_1564() { + let ops = ops_of::( + "typealias A = Int\nclass C where T : Any {\n init {}\n \ + val d by lazy { 1 }\n var p: Int = 0\n get() = field\n \ + set(v) { field = v }\n}\n", + "foo.kt", + ); + for word in ["typealias", "init", "where", "by", "get", "set"] { + assert!( + ops.operators.iter().any(|op| op == word), + "{word}: {:?}", + ops.operators + ); + } + assert!( + ops.operands.iter().any(|op| op == "field"), + "{:?}", + ops.operands + ); + + // The words used as names are operands, not merely non-operators: + // the operator rows above cannot see a name that bills nothing. + let names = ops_of::( + "fun get(x: X) {\n val set = 1\n val by = 2\n x.init(where(set, by))\n}\n", + "foo.kt", + ); + for word in ["init", "where", "by", "get", "set"] { + assert!( + names.operands.iter().any(|op| op == word), + "{word}: {:?}", + names.operands + ); + assert!( + !names.operators.iter().any(|op| op == word), + "{word}: {:?}", + names.operators + ); + } + } + /// #1557 rows shared by the two C++ clones: the declaration keywords /// #1552 left out, one per row, plus the two disambiguator positions /// (`typename T::x`, `x.template f`) that bill the same leaf. @@ -6791,6 +7894,206 @@ end", assert_operator_rows::("foo.tsx", ROWS); } + /// #1563: TypeScript bills its member modifiers as C# and Kotlin bill + /// theirs, each as the keyword leaf and never as the + /// `accessibility_modifier` / `override_modifier` wrapper. The rows + /// after the class members are the same leaves outside a class; the + /// last two are the words used as names, which add no operator. + #[cfg(feature = "typescript")] + const TS_MEMBER_MODIFIER_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}`, `public`, `private`, `protected`, `=` x3, `;` x3 + ( + "visibility", + "class C {\n public a = 1;\n private b = 2;\n protected c = 3;\n}\n", + [7, 11], + ), + // `class`, `extends`, `{}`, `static`, `readonly`, `override`, + // `=` x2, `;` x2 + ( + "static, readonly, override", + "class D extends B {\n static readonly x = 1;\n override y = 2;\n}\n", + [8, 10], + ), + // `abstract` x2, `class`, `{}`, `()`, `:`, `void`, `;` + ( + "abstract", + "abstract class A {\n abstract m(): void;\n}\n", + [7, 8], + ), + // `class`, `{}`, `accessor`, `=`, `;` + ("accessor", "class E {\n accessor q = 1;\n}\n", [5, 5]), + // `class`, `{}` x3, `static` x2, `()` + ( + "static block and method", + "class F {\n static {}\n static m() {}\n}\n", + [4, 7], + ), + // `class`, `{}` x2, `()`, `private`, `readonly`, `:` x2, `number`, + // `,`, `public`, `string` + ( + "parameter properties", + "class G {\n constructor(private readonly p: number, public o: string) {}\n}\n", + [10, 12], + ), + // `type`, `=`, `readonly`, `string`, `[]`, `;` + ("readonly type", "type R = readonly string[];\n", [6, 6]), + // `type`, `=`, `{}`, `-`, `readonly`, `[]`, `in`, `:`, `;` + ( + "mapped type modifier", + "type M = { -readonly [k in K]: T };\n", + [9, 9], + ), + // `type`, `=`, `abstract`, `new`, `()`, `=>`, `void`, `;` + ( + "abstract constructor type", + "type A = abstract new () => void;\n", + [8, 8], + ), + // `let`, `const` x2, `=` x3, `;` x4, `.`, `()`, `{}`, `:` x5, + // `,` x4 — no modifier + ( + "names", + "let readonly = 1;\nobj.static;\nconst a = abstract();\n\ + const o = { public: 1, private: 2, protected: 3, override: 4, accessor: 5 };\n", + [9, 22], + ), + // `class`, `{}` x2, `static`, `=`, `;` x2, `()` — the modifier + // bills once; the field named `static`, the method named + // `readonly` and the field named `public` bill none + ( + "member names", + "class K {\n static static = 1;\n readonly() {}\n public;\n}\n", + [6, 8], + ), + ]; + + #[cfg(feature = "typescript")] + #[test] + fn typescript_member_modifiers_are_operators_1563() { + assert_operator_rows::("foo.ts", TS_MEMBER_MODIFIER_ROWS); + assert_operator_rows::("foo.tsx", TS_MEMBER_MODIFIER_ROWS); + } + + /// #1563: the issue's own fixture, by vocabulary. Every modifier + /// reaches the operator list once, by its word and never by its + /// wrapper's kind name, and the names stay the only operands. + #[cfg(feature = "typescript")] + #[test] + fn typescript_member_modifier_vocabulary_1563() { + const SOURCE: &str = "abstract class C {\n declare x: number;\n \ + public static readonly y = 1;\n private z = 2;\n \ + protected override w = 3;\n abstract m(): void;\n \ + accessor q = 4;\n}\n"; + const OPERATORS: &[&str] = &[ + "()", + ":", + ";", + "=", + "abstract", + "accessor", + "class", + "declare", + "number", + "override", + "private", + "protected", + "public", + "readonly", + "static", + "void", + "{}", + ]; + const OPERANDS: &[&str] = &["1", "2", "3", "4", "C", "m", "q", "w", "x", "y", "z"]; + for (file, ops) in [ + ("foo.ts", ops_of::(SOURCE, "foo.ts")), + ("foo.tsx", ops_of::(SOURCE, "foo.tsx")), + ] { + assert_eq!(ops.operators, OPERATORS, "{file}"); + assert_eq!(ops.operands, OPERANDS, "{file}"); + } + } + + /// #1563: the modifier words used as names are operands and never + /// operators, in an expression, a member access, a call, an object + /// key and a class member name. No getter edit makes this fail: the + /// modifier arms key on the keyword kinds, and the grammar emits an + /// identifier for each name. It pins that grammar property, so a + /// grammar bump or a text-keyed arm that bills a name trips it. + #[cfg(feature = "typescript")] + #[test] + fn typescript_modifier_words_as_names_are_operands_1563() { + const SOURCE: &str = "let readonly = 1;\nobj.static;\nconst a = abstract();\n\ + const o = { public: 1, private: 2, protected: 3, override: 4, \ + accessor: 5 };\nclass K { static() {} }\n"; + const KEYWORDS: &[&str] = &[ + "readonly", + "static", + "abstract", + "public", + "private", + "protected", + "override", + "accessor", + ]; + assert_keywords_are_operands_only::(SOURCE, "foo.ts", KEYWORDS); + assert_keywords_are_operands_only::(SOURCE, "foo.tsx", KEYWORDS); + } + + /// #1563: JavaScript's `static` bills as TypeScript's does, in a + /// field, a method and a `static { … }` block. `static get` with the + /// name on the next line is one token, so it is one operator. The + /// last row is the word used as names, which adds no operator. + #[cfg(any(feature = "javascript", feature = "mozjs"))] + const JS_STATIC_ROWS: &[(&str, &str, [u64; 2])] = &[ + // `class`, `{}`, `static`, `=`, `;` + ("static field", "class K {\n static x = 1;\n}\n", [5, 5]), + // `class`, `{}` x3, `static` x2, `()` + ( + "static method and block", + "class K {\n static m() {}\n static {}\n}\n", + [4, 7], + ), + // `class`, `{}` x2, `static`, `get`, `()` + ( + "static getter", + "class K {\n static get y() {}\n}\n", + [5, 6], + ), + // `class`, `{}` x2, `static get`, `()` + ( + "static get before a newline", + "class K {\n static get\n y() {}\n}\n", + [4, 5], + ), + // `.`, `;` x3, `()`, `var`, `=`, `{}`, `:` — no `static` + ( + "names", + "obj.static;\nstatic();\nvar o = { static: 1 };\n", + [7, 9], + ), + ]; + + #[cfg(any(feature = "javascript", feature = "mozjs"))] + #[test] + fn javascript_static_is_an_operator_1563() { + const NAMES: &str = "obj.static;\nstatic();\nvar o = { static: 1 };\n\ + class K { static static = 1; }\n"; + #[cfg(feature = "javascript")] + { + assert_operator_rows::("foo.js", JS_STATIC_ROWS); + let ops = ops_of::(NAMES, "foo.js"); + assert!(ops.operands.iter().any(|o| o == "static"), "{ops:?}"); + assert!(ops.operators.iter().any(|o| o == "static"), "{ops:?}"); + } + #[cfg(feature = "mozjs")] + { + assert_operator_rows::("foo.js", JS_STATIC_ROWS); + let ops = ops_of::(NAMES, "foo.js"); + assert!(ops.operands.iter().any(|o| o == "static"), "{ops:?}"); + assert!(ops.operators.iter().any(|o| o == "static"), "{ops:?}"); + } + } + #[cfg(feature = "bash")] #[test] fn bash_operators_and_operands() { @@ -11066,13 +12369,13 @@ S s; int a = s[1]; auto t = co_await s; #[cfg(any(feature = "cpp", feature = "mozcpp"))] #[track_caller] fn assert_cpp_operator_names(label: &str) { - // Operators (n1 = 17, N1 = 40): `operator` 5 (four + // Operators (n1 = 18, N1 = 41): `operator` 5 (four // `operator_name`s and the `operator_cast`); the names `[]` 1, // `()` 1, `""` 1; `co_await` 2 (the name and the expression); // the `(` of a parameter list 5; `;` 9; `int` 3; `long` 3; // `bool` 2; `=` 2; `{` 1; `void` 1; `unsigned` 1; `::` 1; the - // subscript `[` 1; `struct` 1 (#1552). `auto` and `const` are in - // no arm. + // subscript `[` 1; `struct` 1 (#1552); the trailing `const` 1 + // (#1578). `auto` is in no arm. // // Operands (n2 = 6, N2 = 9): `s` 3, `S` 2 (the qualifier in // `S::operator bool` is a namespace identifier, which #1096 @@ -11080,7 +12383,7 @@ S s; int a = s[1]; auto t = co_await s; // // Before #1296: (12, 30, 6, 9) with `struct`, the five operator // kinds billed nowhere. - assert_halstead_counts::(CPP_OPERATOR_NAMES, "ops.cpp", [17, 40, 6, 9], label); + assert_halstead_counts::(CPP_OPERATOR_NAMES, "ops.cpp", [18, 41, 6, 9], label); // `operator[]` / `operator()` are kinds of their own, so they sit // beside the subscript `[` and the parameter-list `(` as two more diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 1d76d550c..6ed5f3ad7 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -78,6 +78,9 @@ mod cpp_alternative_tokens_tests; #[path = "elixir_named_operator_tests.rs"] mod elixir_named_operator_tests; #[cfg(test)] +#[path = "elixir_spelling_parity_tests.rs"] +mod elixir_spelling_parity_tests; +#[cfg(test)] #[path = "perl_keyword_key_tests.rs"] mod perl_keyword_key_tests; diff --git a/src/metrics/nexits.rs b/src/metrics/nexits.rs index 347c1a496..307c07177 100644 --- a/src/metrics/nexits.rs +++ b/src/metrics/nexits.rs @@ -20,6 +20,7 @@ use std::fmt; use crate::checker::Checker; +use crate::lang_helpers::perl::perl_bareword_is_key; use crate::macros::implement_metric_trait; use crate::*; @@ -136,7 +137,16 @@ where { /// Walk `node` and update `stats` with this metric for the language /// implementing the trait. - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats); + /// + /// `ancestors` is `node`'s chain as the metric walk recorded it, + /// sibling cache included; Perl reads it to tell an auto-quoted + /// `die =>` key from the builtin (#1585). + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ); } // Bumps `stats.exit` whenever the current node matches any of the @@ -145,7 +155,12 @@ where macro_rules! impl_exit_match_kinds { ($code:ty, $lang:ident, [$($kind:ident),+ $(,)?]) => { impl Exit for $code { - fn compute<'a>(node: &Node<'a>, _code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + _code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if matches!(node.kind_id().into(), $($lang::$kind)|+) { stats.exit += 1; } @@ -229,7 +244,12 @@ impl_exit_match_kinds!( ); impl Exit for RustCode { - fn compute<'a>(node: &Node<'a>, _code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + _code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { // Count only explicit `return` and `?` (TryExpression). The // implicit final-expression path is NOT an exit — peer-language // impls have the same convention. See #243 for the prior bug @@ -245,7 +265,12 @@ impl Exit for RustCode { } impl Exit for CsharpCode { - fn compute<'a>(node: &Node<'a>, _code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + _code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if matches!( node.kind_id().into(), Csharp::ReturnStatement @@ -267,7 +292,12 @@ impl Exit for GoCode { // `selector_expression`) means a package-qualified call like // `foo.panic()` — a user function, not the built-in — is not counted, // mirroring how Bash matches the bare builtin command name. - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if matches!(node.kind_id().into(), Go::ReturnStatement) { stats.exit += 1; } else if node.kind_id() == Go::CallExpression @@ -307,19 +337,23 @@ impl Exit for PerlCode { // plain `identifier`, never a bareword, so a user method named // `die` is not counted. // - // Known artifact: a fat-comma hash key auto-quotes its bareword in - // Perl, but the grammar still emits `call_expression_with_bareword` - // for it, so `(die => 1)` counts as an exit. Gating on the `=>` - // that follows would need a forward sibling lookup from inside the - // metric walk, which #1096 took out of these bodies; the same node - // already counts as an ABC branch (`src/metrics/abc/perl.rs`), so - // the artifact is accepted rather than paid for here. - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + // - An auto-quoted key (`(die => 1)`) is a string, though the + // grammar parses it as a bareword call (#1571 upstream). ABC and + // `find --type call` skip it, and so does this (#1585). The text + // test runs first: the key test reads a sibling, which the walk's + // `SiblingCache` keeps linear under a flat `ERROR`. + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { let is_abrupt_exit_builtin = node.kind_id() == Perl::CallExpressionWithBareword && matches!( node.utf8_text(code), Some("die" | "exit" | "CORE::die" | "CORE::exit") - ); + ) + && !perl_bareword_is_key(node, ancestors); if node.kind_id() == Perl::ReturnExpression || is_abrupt_exit_builtin { stats.exit += 1; } @@ -327,7 +361,12 @@ impl Exit for PerlCode { } impl Exit for KotlinCode { - fn compute<'a>(node: &Node<'a>, _code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + _code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if matches!( node.kind_id().into(), Kotlin::ReturnExpression | Kotlin::ThrowExpression @@ -346,7 +385,12 @@ impl Exit for LuaCode { // `os.exit`. Count them as exits alongside `return`. Matching the exact // callee text means a user call such as `foo()` or `myError()` is not // counted, mirroring how Bash/Elixir match the bare builtin name. - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if node.kind_id() == Lua::ReturnStatement { stats.exit += 1; } else if node.kind_id() == Lua::FunctionCall @@ -359,7 +403,12 @@ impl Exit for LuaCode { } impl Exit for BashCode { - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { // Bash has no `return_statement` node: `return` and `exit` are // ordinary builtins parsed as `Bash::Command` whose `name` field // points at a `Bash::CommandName`. Identify them by comparing the @@ -374,7 +423,12 @@ impl Exit for BashCode { } impl Exit for TclCode { - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { // Tcl has no return keyword node; `return` is a generic Command whose // name field is a simple_word with text "return" — the same // leading-word resolution the Cognitive / Cyclomatic `switch` and @@ -399,7 +453,12 @@ impl Exit for TclCode { } impl Exit for IrulesCode { - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { // Like Tcl, iRules has no `return` keyword node — `return` is a // generic Command (it is not among the grammar's `_builtin` // commands). The bare name word can surface as either `simple_word` @@ -441,7 +500,12 @@ impl Exit for PhpCode { // text — brittle and likely to false-match user-defined `die` // functions. Modern PHP idiom favors `throw new Exception()` over // `die`, so leaving this asymmetric is acceptable. - fn compute<'a>(node: &Node<'a>, _code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + _code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if matches!( node.kind_id().into(), Php::ReturnStatement | Php::YieldExpression | Php::ThrowExpression | Php::ExitStatement @@ -492,7 +556,12 @@ impl Exit for RubyCode { // The callee is inspected before the receiver: every call has a // `method`, and the text test rejects nearly all of them, so the // receiver lookup runs only for the handful that spell a builtin. - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if matches!(node.kind_id().into(), Ruby::Return | Ruby::Return2) { stats.exit += 1; } else if Self::is_call(node) @@ -514,7 +583,12 @@ impl Exit for ElixirCode { // `raise`, `reraise`, or `exit`, all of which surface as `Call` // nodes whose target is an `Identifier` whose text spells the // keyword. Mirrors the Bash/Tcl pattern of comparing target text. - fn compute<'a>(node: &Node<'a>, code: &'a [u8], stats: &mut Stats) { + fn compute<'a>( + node: &Node<'a>, + code: &'a [u8], + _ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + ) { if node.kind_id() == Elixir::Call && let Some(target) = node.child_by_field_name("target") && target.kind_id() == Elixir::Identifier @@ -542,6 +616,7 @@ impl Exit for ElixirCode { mod tests { use crate::test_support::{ check_func_space_only_shim, check_metrics_only_shim, child_space, function_space, + metrics_verbatim, parse_named, }; use super::*; @@ -1602,6 +1677,105 @@ mod tests { ); } + /// The root space's `nexits` sum for the Perl file `source`. + #[cfg(feature = "perl")] + fn perl_nexits(source: &str) -> u64 { + metrics_verbatim( + LANG::Perl, + source.as_bytes(), + MetricsOptions::default().with_only(&[Metric::Nexits]), + ) + .nexits + .nexits_sum() + } + + /// Each auto-quoted `die` / `exit` key scores exactly as its quoted + /// twin does (#1585). tree-sitter-perl reads `die` before `=>` as a + /// bareword call (#1571 upstream), so the `=>` rows each counted one + /// exit per key before the fix: the first row read 3. The subscript + /// and dash rows never parse to a `die` / `exit` bareword at all + /// (`$h{die}` is an `identifier`, `-exit` a file test on `xit`); they + /// pin that the gate keeps them at the twin's reading. `- die =>` + /// negates the quoted string, and a newline before the `=>` does not + /// stop it quoting. + #[cfg(feature = "perl")] + #[test] + fn perl_exit_key_scores_like_its_quoted_twin() { + let rows: [(&str, u64); 9] = [ + ( + "sub f { my %h = ('exit' => 1, 'die' => 2); return 1; }\n", + 1, + ), + ("sub f { my %h = (- 'die' => 1); return 1; }\n", 1), + ("sub f { my %h = ('exit'\n => 1); return 1; }\n", 1), + ( + "sub f { my $r = { 'die' => 1, 'exit' => 2 }; return $r; }\n", + 1, + ), + ("sub f { g('exit' => 1); return; }\n", 1), + ( + "sub f { my %h = ('-die' => 1, '-exit' => 2); return 1; }\n", + 1, + ), + ("sub f { my $v = $h{'die'}; return $v; }\n", 1), + ("sub f { my $v = $h{'-exit'}; return $v; }\n", 1), + ("sub f { my %h = ('exit' => 1); exit 2; }\n", 1), + ]; + for (twin, want) in rows { + let key = twin.replace('\'', ""); + assert_eq!(perl_nexits(twin), want, "`{twin}` moved; re-derive the row"); + assert_eq!(perl_nexits(&key), want, "`{key}` must score like `{twin}`"); + } + } + + /// A `die` / `exit` that is no auto-quoted key keeps its exit: every + /// call spelling, a call inside a subscript, a call before `=>` + /// (`die()` takes arguments, so `=>` does not quote it), a value + /// after `=>`, and the `CORE::` spelling, which `=>` does not quote + /// either since `::` is no word character. Every value is `main`'s, + /// measured before #1585. + #[cfg(feature = "perl")] + #[test] + fn perl_exit_outside_a_key_keeps_its_exit() { + let rows: [(&str, u64); 9] = [ + ("sub f { die \"x\"; }\n", 1), + ("sub f { exit 1; }\n", 1), + ("sub f { exit; }\n", 1), + ("sub f { open(my $fh, '<', 'p') or die \"x\"; }\n", 1), + ("sub f { my $v = $h{die()}; }\n", 1), + ("sub f { my $v = $h{exit()}; }\n", 1), + ("sub f { my %h = (die() => 1); }\n", 1), + ("sub f { my %h = (x => die); }\n", 1), + ("sub f { my %h = (CORE::die => 1); }\n", 1), + ]; + for (source, want) in rows { + assert_eq!(perl_nexits(source), want, "`{source}` changed its reading"); + } + } + + /// A file whose tail fails to parse leaves every key and call in it a + /// direct child of one flat `ERROR` root. The key test reads the + /// token after each `die`, which the metric walk's `SiblingCache` + /// answers (#1584); a walk that lent the cache without recording the + /// `ERROR` would trip the debug assertion in that lookup. The reading + /// is the twelve `exit` calls, as the quoted twin's is. + #[cfg(feature = "perl")] + #[test] + fn perl_exit_key_in_a_flat_error_scores_like_its_twin() { + // The width from which a walk records a node's children. + use big_code_analysis_ast::node::WIDE_PARENT; + let key = format!("my @r = ({}", "{\ndie => 1,\nexit,\n".repeat(12)); + let twin = key.replace("die =>", "'die' =>"); + let ast = parse_named(LANG::Perl, "foo.pl", &key); + let root = ast.as_tree_sitter().root_node(); + assert!( + root.is_error() && root.child_count() > WIDE_PARENT, + "the fixture must parse to a wide flat ERROR root" + ); + assert_eq!(perl_nexits(&twin), 12); + assert_eq!(perl_nexits(&key), 12); + } + #[cfg(feature = "typescript")] #[test] fn tsx_function_with_returns() { diff --git a/src/metrics/perl_keyword_key_tests.rs b/src/metrics/perl_keyword_key_tests.rs index e16500af1..fa2b5df0a 100644 --- a/src/metrics/perl_keyword_key_tests.rs +++ b/src/metrics/perl_keyword_key_tests.rs @@ -1,7 +1,8 @@ //! An auto-quoted `and` or `not` key scores like any other key, and the //! `and` and `not` operators still score as operators (#1539, #1541). //! The same holds for a `-bareword` key and the file tests the grammar -//! mistakes it for (#1545). +//! mistakes it for (#1545), and for every other bareword key, which the +//! grammar reads as a call (#1572). //! //! Perl quotes a bareword before `=>` and alone in a hash subscript, so //! `$h{and}` and `(not => 1)` hold a string key and no operator. @@ -416,10 +417,11 @@ const DASH_KEY_ROWS: [(&str, &str, [u64; 4], Halstead); 20] = [ [2, 1, 0, 0], [5, 5, 3, 4], ), + // The `ext` key scored a branch for "calling" it until #1572. ( - "f('-text' => 1, ext => 2);", + "f('-text' => 1, 'ext' => 2);", "f(-text => 1, ext => 2);", - [2, 2, 0, 0], + [2, 1, 0, 0], [6, 7, 5, 6], ), ( @@ -499,9 +501,9 @@ const DASH_KEY_ROWS: [(&str, &str, [u64; 4], Halstead); 20] = [ /// swallowing the `=>`, and `-not` as a `-` applied to the keyword. /// Before the fix each `-foo` billed the operand `oo` and an ABC branch /// for calling it, and each `-not` / `-and` billed a `-` operator. A key -/// whose letter is no file-test letter (`-name`) is not among them: it -/// parses as a `-` applied to a bareword, which still bills the `-` and -/// an ABC branch for the word, as every bareword key does. +/// whose letter is no file-test letter (`-name`) parses as a `-` applied +/// to a bareword instead, and is pinned by +/// `perl_bareword_key_scores_like_its_quoted_twin`. /// /// The rows pairing a key with a second key equal to the misparsed /// word (`ext`, `not`) or to itself (`-x` twice) pin the operand's @@ -530,7 +532,9 @@ fn perl_dash_key_scores_like_its_quoted_twin() { /// A `-and` / `-not` key keeps its sign when whitespace separates the /// two: Perl keys `(- not => 1)` as `"-not"`, so it must not share a /// vocabulary entry with the plain `not` key beside it. The twins' -/// values are `DASH_KEY_ROWS`' for the glued spelling. +/// values are `DASH_KEY_ROWS`' for the glued spelling. A `- name` key +/// keeps its sign too (#1572); it is spelled as written, space and all, +/// which the counts cannot see, since only one signed key is present. #[cfg(feature = "perl")] #[test] fn perl_spaced_dash_key_keeps_its_sign() { @@ -542,6 +546,16 @@ fn perl_spaced_dash_key_keeps_its_sign() { "f('-not' => 1, not => 2);", "f(- # c\n not => 1, not => 2);", ), + ( + "f('-name' => 1, 'name' => 2);", + "f(- name => 1, name => 2);", + ), + // The slice from the sign carries the comment, so the key is + // still one operand apart from `name`. + ( + "f('-name' => 1, 'name' => 2);", + "f(- # c\n name => 1, name => 2);", + ), ] { let want = measure_with_branches(twin); assert_eq!( @@ -642,3 +656,479 @@ fn perl_file_test_outside_a_key_keeps_its_reading() { ); } } + +/// `(-foo, => 1)` pins the grammar's misparse, not Perl's reading. +/// +/// Perl reads `-foo` here as the string `-foo`: `perl -e 'my @c = +/// (-foo, => 1); print "@c"'` prints `-foo 1`. tree-sitter-perl instead +/// parses `-foo` as a `file_handle_operator` applied to the bareword +/// `oo`, and the stray `,` ends the key position, so the `4, 4` +/// operands below come from billing `oo`. `main` read the key `-foo`, +/// because its `=>` lookup skipped the `ERROR` the stray `,` recovers +/// into; since #1572 the shape reads as the grammar does, as +/// `(name, => 1)` does (see `is_key_position`). The value is pinned so +/// a change to it is seen, not endorsed as correct. +// FIXME(#1571): bill `-foo` as the one string operand Perl reads. +#[cfg(feature = "perl")] +#[test] +fn perl_dash_key_before_stray_comma_pins_the_grammar_misparse_1571() { + assert_eq!( + measure_with_branches("my @c = (-foo, => 1);"), + ([2, 1, 0, 0], [8, 8, 4, 4]), + "the grammar's `oo` reading moved; re-derive against #1571" + ); +} + +/// `(quoted_twin, bareword_key, [cyclomatic, branches, conditions, +/// cognitive], halstead)` rows for +/// `perl_bareword_key_scores_like_its_quoted_twin`. Every value is the +/// twin's, measured on `main` before #1572. +#[cfg(feature = "perl")] +const BAREWORD_KEY_ROWS: [(&str, &str, [u64; 4], Halstead); 23] = [ + ( + "my %h = ('name' => 1); return 1;", + "my %h = (name => 1); return 1;", + [2, 0, 0, 0], + [8, 9, 4, 5], + ), + ( + "my %h = ('-name' => 1); return 1;", + "my %h = (-name => 1); return 1;", + [2, 0, 0, 0], + [8, 9, 4, 5], + ), + ( + "my %h; return $h{'-name'};", + "my %h; return $h{-name};", + [2, 0, 0, 0], + [6, 8, 4, 4], + ), + ( + "my %h; return $h{ '-name' };", + "my %h; return $h{ -name };", + [2, 0, 0, 0], + [6, 8, 4, 4], + ), + ( + "my $r; return $r->{'-name'};", + "my $r; return $r->{-name};", + [2, 0, 0, 0], + [7, 10, 3, 4], + ), + ( + "my %h; return @h{'-a'};", + "my %h; return @h{-a};", + [2, 0, 0, 0], + [5, 7, 4, 4], + ), + ( + "my %h; if ($h{'-name'}) { return 1; } return 0;", + "my %h; if ($h{-name}) { return 1; } return 0;", + [3, 0, 1, 1], + [8, 13, 6, 6], + ), + ( + "my $r = { 'a' => { 'b' => 1 } }; return $r;", + "my $r = { a => { b => 1 } }; return $r;", + [2, 0, 0, 0], + [8, 13, 5, 6], + ), + // A named-argument call scores its callee alone. + ( + "new('name' => $x, 'size' => 3);", + "new(name => $x, size => 3);", + [2, 1, 0, 0], + [7, 8, 6, 6], + ), + ( + "foo 'name' => 1;", + "foo name => 1;", + [2, 1, 0, 0], + [4, 4, 4, 4], + ), + ( + "$o->m('-name' => 1);", + "$o->m(-name => 1);", + [2, 1, 0, 0], + [7, 7, 5, 5], + ), + // Perl quotes a word across a comment or line break before `=>`. + ( + "my %h = ('name' # c\n => 1); return 1;", + "my %h = (name # c\n => 1); return 1;", + [2, 0, 0, 0], + [8, 9, 4, 5], + ), + // The spelling rows: a signed and a plain key are two operands, and + // two signed keys one, as their twins are. Billed as `name`, the + // first pair merges (n2 4) and the second still matches; billed as + // `-` + `name`, both also bill a `-` operator. + ( + "f('-name' => 1, 'name' => 2);", + "f(-name => 1, name => 2);", + [2, 1, 0, 0], + [6, 7, 5, 6], + ), + ( + "f('-name' => 1, '-name' => 2);", + "f(-name => 1, -name => 2);", + [2, 1, 0, 0], + [6, 7, 4, 6], + ), + // A builtin's name is quoted like any other word. + ( + "my %h = ('shift' => 1, 'time' => 2); return 1;", + "my %h = (shift => 1, time => 2); return 1;", + [2, 0, 0, 0], + [9, 11, 6, 7], + ), + ( + "my @a = ('a' => 'b' => 1); return @a;", + "my @a = (a => b => 1); return @a;", + [2, 0, 0, 0], + [8, 10, 5, 6], + ), + // A bareword *value* is no key, and still scores its call. + ( + "my @a = ('a' => foo); return @a;", + "my @a = (a => foo); return @a;", + [2, 1, 0, 0], + [8, 9, 4, 5], + ), + // `=>` quotes the word right before it whatever expression the word + // closes, which the grammar nests as that expression's last operand. + // A subtraction's `-` stays an operator. + ( + "my @a = (1 - 'name' => 2); return @a;", + "my @a = (1 - name => 2); return @a;", + [2, 0, 0, 0], + [9, 10, 5, 6], + ), + ( + "my @a = ($x . 'name' => 2); return @a;", + "my @a = ($x . name => 2); return @a;", + [2, 0, 0, 0], + [10, 11, 5, 6], + ), + ( + "my @a = (f(1), 'name' => 2); return @a;", + "my @a = (f(1), name => 2); return @a;", + [2, 1, 0, 0], + [9, 11, 5, 7], + ), + // After a keyword or file-test key, whose recovery leaves an + // `ERROR` in the same list. + ( + "f(not => 1, 'name' => 2);", + "f(not => 1, name => 2);", + [2, 1, 0, 0], + [6, 7, 5, 6], + ), + ( + "f(and => 1, '-name' => 2);", + "f(and => 1, -name => 2);", + [2, 1, 0, 0], + [6, 7, 5, 6], + ), + ( + "f('-x' => 1, 'name' => 2);", + "f(-x => 1, name => 2);", + [2, 1, 0, 0], + [6, 7, 5, 6], + ), +]; + +/// Each bareword key below scores exactly as its quoted twin does, in +/// all four metrics (#1572). tree-sitter-perl reads `name` before `=>` +/// as a call of `name`, and `-name` there or alone in a subscript as a +/// `-` applied to that call. Before the fix each key scored an ABC +/// branch, and each `-name` billed a `-` operator and the unsigned +/// operand `name`. Cyclomatic and cognitive were never affected, which +/// the decision columns pin. +#[cfg(feature = "perl")] +#[test] +fn perl_bareword_key_scores_like_its_quoted_twin() { + for (twin, key, decisions, halstead) in BAREWORD_KEY_ROWS { + assert_eq!(twin.replace('\'', ""), key, "`{key}` drifted from its twin"); + let want = measure_with_branches(twin); + assert_eq!( + want, + (decisions, halstead), + "`{twin}` moved; re-derive the row" + ); + assert_eq!( + measure_with_branches(key), + want, + "`{key}` must score like `{twin}`" + ); + } +} + +/// A bareword that is no auto-quoted key keeps the call, and a `-` that +/// signs none keeps its operator: a word followed by anything but `=>`, +/// a package-qualified word (`=>` quotes only letters, digits and +/// underscores), a word with arguments, a call inside a subscript, a +/// `-bareword` that is only part of one, a word a stray `,` separates +/// from the `=>` (the grammar hides that `,` in an extra `ERROR`, and +/// the `=>` then quotes the `,`, which is nothing), and a unary minus on +/// a variable. Every value is `main`'s, measured before #1572. +#[cfg(feature = "perl")] +#[test] +fn perl_bareword_outside_a_key_keeps_its_call() { + let rows: [(&str, [u64; 4], Halstead); 11] = [ + ("foo(-name);", [2, 2, 0, 0], [5, 5, 3, 3]), + ( + "my @a = (name, => 1); return @a;", + [2, 1, 0, 0], + [9, 10, 4, 5], + ), + ( + "my @a = (-name, => 1); return @a;", + [2, 1, 0, 0], + [10, 11, 4, 5], + ), + ("return -name;", [2, 1, 0, 0], [5, 5, 2, 2]), + ("my @a = (name, 1); return @a;", [2, 1, 0, 0], [8, 9, 4, 5]), + ( + "my @a = (Foo::Bar => 1); return @a;", + [2, 1, 0, 0], + [9, 10, 5, 6], + ), + ( + "my @a = (name() => 1); return @a;", + [2, 1, 0, 0], + [8, 10, 4, 5], + ), + ("my %h; return $h{foo()};", [2, 1, 0, 0], [7, 9, 4, 4]), + ( + "my %h; return $h{-name . 'x'};", + [2, 1, 0, 0], + [8, 10, 5, 5], + ), + ("my %h; return $h{1 - name};", [2, 1, 0, 0], [7, 9, 5, 5]), + ("my $y = -$x; return $y;", [2, 0, 0, 0], [8, 11, 3, 4]), + ]; + for (body, decisions, halstead) in rows { + assert_eq!( + measure_with_branches(body), + (decisions, halstead), + "`{body}` changed its reading" + ); + } +} + +/// A file whose tail fails to parse leaves every key, chain and sign in +/// it a direct child of one flat `ERROR` root, and the key and misparse +/// lookups ask for a sibling in that `ERROR` once per child. Both walks +/// lend the `SiblingCache` that answers them from the children they +/// recorded rather than by a scan from the `ERROR`'s start (#1584); a +/// walk that stopped recording trips the debug assertion in the lookup +/// here, since the answers themselves would not change. Every value is +/// `main`'s, measured before #1584: twelve `last` calls, the `name` +/// keys none, and `-a` / `-and` billed whole. `find` and `count` walk +/// with `act_on_node`, which lends the same cache. +#[cfg(feature = "perl")] +#[test] +fn perl_flat_error_reads_siblings_from_the_walk() { + // The width from which a walk records a node's children. + use big_code_analysis_ast::node::WIDE_PARENT; + let body = "{\nlast,\n$a && $b,\n- and,\n-a => 1,\nname => 1,\n$c > $d && $e,\n"; + let source = format!("my @r = ({}", body.repeat(12)); + let ast = Ast::parse(Source::new(LANG::Perl, source.as_bytes())).expect("perl is enabled"); + let root = ast.as_tree_sitter().root_node(); + assert!( + root.is_error() && root.child_count() > WIDE_PARENT, + "the fixture must parse to a wide flat ERROR root" + ); + + let space = ast + .metrics(MetricsOptions::default()) + .expect("metrics yields a root space"); + let (abc, halstead) = (&space.metrics.abc, &space.metrics.halstead); + assert_eq!( + ( + abc.branches_sum(), + abc.conditions_sum(), + abc.assignments_sum() + ), + (12, 48, 1) + ); + assert_eq!( + [ + halstead.unique_operators(), + halstead.total_operators(), + halstead.unique_operands(), + halstead.total_operands(), + ], + [9, 207, 11, 133] + ); + + let ops = ast.ops().expect("ops yields a root space"); + assert_eq!( + ops.operands, + [ + "$a", "$b", "$c", "$d", "$e", "-a", "-and", "1", "@r", "last", "name" + ] + ); + assert_eq!( + ops.operators, + ["$", "&&", "()", ",", "=", "=>", ">", "my", "{}"] + ); + + // The `call` filter of `find` and `count` asks the same key test of + // every bareword, through the walk `act_on_node` keeps (#1580): the + // twelve `last` calls, as ABC counts them, and no `name` key. + assert_eq!(ast.count(&["call".to_owned()]).0, 12); +} + +/// What `bca find --type call` reports on `source`: each node's kind and +/// text, in source order, and the root space's ABC branches. `is_call` +/// and ABC's branch arm share `perl_bareword_is_call`, so the two must +/// count the same calls (grammar-dispatch §7) — Perl has no other kind +/// of ABC branch. +#[cfg(feature = "perl")] +fn perl_calls(source: &str) -> (Vec<(String, String)>, usize) { + let ast = Ast::parse(Source::new(LANG::Perl, source.as_bytes())).expect("perl is enabled"); + let found = ast + .find(&["call".to_owned()]) + .expect("find is infallible") + .iter() + .map(|node| { + ( + node.kind().to_owned(), + source[node.start_byte()..node.end_byte()].to_owned(), + ) + }) + .collect::>(); + assert_eq!( + ast.count(&["call".to_owned()]).0, + found.len(), + "`count` and `find` disagree on `{source}`" + ); + let branches = ast + .metrics(MetricsOptions::default()) + .expect("metrics yields a root space") + .metrics + .abc + .branches_sum(); + let branches = usize::try_from(branches).expect("a fixture's branch count fits a usize"); + (found, branches) +} + +/// #1580's fixture: three auto-quoted keys the grammar parses as +/// bareword calls, and one call whose name it parses as a second. Before +/// the fix `find` reported all five; `foo(1)` is the only call. +#[cfg(feature = "perl")] +#[test] +fn perl_find_call_reports_one_node_per_call_1580() { + let source = "sub d { my %h = (name => 1, -size => 2); my $y = $h{-name}; return foo(1); }\n"; + let (found, branches) = perl_calls(source); + assert_eq!( + found, + [( + "call_expression_with_args_with_brackets".to_owned(), + "foo(1)".to_owned() + )] + ); + assert_eq!(branches, 1); +} + +/// Each call spelling is one call, reported on the node present for +/// that spelling: the wrapper where the grammar builds one around the +/// name, else the bareword. `foo(1` is the error-recovery shape with no +/// wrapper at all, so its bareword is the call; dropping bareword calls +/// instead of gating them would lose it, and `foo;` and `&foo;` with it +/// (grammar-dispatch §6). +#[cfg(feature = "perl")] +#[test] +fn perl_each_call_spelling_is_one_call_1580() { + const BRACKETS: &str = "call_expression_with_args_with_brackets"; + const SPACED: &str = "call_expression_with_spaced_args"; + const BAREWORD: &str = "call_expression_with_bareword"; + const METHOD: &str = "method_invocation"; + let rows: [(&str, &[(&str, &str)]); 14] = [ + ("foo(1);", &[(BRACKETS, "foo(1)")]), + ("foo();", &[(BRACKETS, "foo()")]), + ("&foo(1);", &[(BRACKETS, "&foo(1)")]), + ("Foo::bar(1);", &[(BRACKETS, "Foo::bar(1)")]), + ("foo 1;", &[(SPACED, "foo 1")]), + ("print \"x\";", &[(SPACED, "print \"x\"")]), + ( + "sort { $a <=> $b } @x;", + &[(SPACED, "sort { $a <=> $b } @x")], + ), + ("$obj->m(1);", &[(METHOD, "$obj->m(1)")]), + ( + "&$code(1);", + &[("call_expression_with_variable", "&$code(1)")], + ), + ( + "__SUB__->(1);", + &[("call_expression_recursive", "__SUB__->(1)")], + ), + ("foo;", &[(BAREWORD, "foo")]), + ("&foo;", &[(BAREWORD, "&foo")]), + // A bareword argument is a call of its own, not the callee. + ("foo(bar);", &[(BRACKETS, "foo(bar)"), (BAREWORD, "bar")]), + ("foo(1", &[(BAREWORD, "foo")]), + ]; + for (source, expected) in rows { + let (found, branches) = perl_calls(&format!("{source}\n")); + let expected = expected + .iter() + .map(|&(kind, text)| (kind.to_owned(), text.to_owned())) + .collect::>(); + assert_eq!(found, expected, "`{source}`"); + assert_eq!(branches, found.len(), "`{source}`: ABC disagrees"); + } +} + +/// An auto-quoted key is a string, so it reports the calls its quoted +/// twin does — none of its own. A word Perl does not quote keeps its +/// call: a package-qualified word, one with arguments, and a value. +/// Every count is checked against ABC's branches as well. +#[cfg(feature = "perl")] +#[test] +fn perl_bareword_key_is_no_call_1580() { + let rows = [ + ("my %h = (name => 1);", "my %h = ('name' => 1);", 0), + ("my %h = (-name => 1);", "my %h = ('-name' => 1);", 0), + ("my %h = (-size => 1);", "my %h = ('-size' => 1);", 0), + ("my $y = $h{-name};", "my $y = $h{'-name'};", 0), + ("my $y = $h{-size};", "my $y = $h{'-size'};", 0), + ("my %h = (1 - name => 2);", "my %h = (1 - 'name' => 2);", 0), + ("new(name => $x);", "new('name' => $x);", 1), + ("my %h = (Foo::Bar => 1);", "my %h = (Foo::Bar() => 1);", 1), + ( + "my %h = (-foo::bar => 1);", + "my %h = (-foo::bar() => 1);", + 1, + ), + ("my %h = (name() => 1);", "my %h = (name(1) => 1);", 1), + ("my %h = (k => name);", "my %h = ('k' => name());", 1), + ]; + for (key, twin, calls) in rows { + let (found, branches) = perl_calls(&format!("{key}\n")); + let (twin_found, _) = perl_calls(&format!("{twin}\n")); + assert_eq!(found.len(), calls, "`{key}` reported {found:?}"); + assert_eq!(found.len(), twin_found.len(), "`{key}` against `{twin}`"); + assert_eq!(branches, calls, "`{key}`: ABC disagrees"); + } +} + +#[cfg(feature = "perl")] +#[test] +fn perl_negated_qualified_calls_keep_their_operand_names() { + let source = "my %h = (-foo::bar => 1, -foo::baz => 2);"; + let ast = Ast::parse(Source::new(LANG::Perl, source.as_bytes())).expect("perl is enabled"); + assert_eq!(ast.count(&["call".to_owned()]).0, 2); + let metrics = ast + .metrics(MetricsOptions::default()) + .expect("metrics has a root"); + assert_eq!(metrics.metrics.abc.branches_sum(), 2); + assert_eq!(metrics.metrics.halstead.unique_operands(), 6); + assert_eq!( + ast.ops().expect("ops has a root").operands, + ["%h", "1", "2", "bar", "baz", "oo"] + ); +} diff --git a/src/ops.rs b/src/ops.rs index 137982c53..6939e34c4 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -11,7 +11,7 @@ use std::borrow::Cow; use crate::checker::Checker; use crate::error::MetricsError; use crate::getter::Getter; -use crate::node::{Ancestors, Node}; +use crate::node::{Ancestors, Node, SiblingCache}; use crate::spaces::{SpaceKind, line_span, push_children}; use crate::halstead::{Halstead, HalsteadMaps}; @@ -310,6 +310,8 @@ pub(crate) fn ops_inner( // maintained by the same truncate/push rule as // `spaces::compute::metrics_inner` (#1084). let mut chain: Vec> = Vec::new(); + // As in `metrics_inner`: the children of each wide node on `chain`. + let mut siblings = SiblingCache::default(); let mut state_stack: Vec = Vec::new(); let mut last_level = 0; @@ -330,7 +332,7 @@ pub(crate) fn ops_inner( last_level = level; } - let ancestors = Ancestors::checked(&chain, &node); + let ancestors = Ancestors::checked(&chain, &node).with_siblings(&siblings); // Same predicate `spaces::compute::metrics_inner` opens on, so // the two walks agree on which nodes become spaces. The @@ -366,9 +368,7 @@ pub(crate) fn ops_inner( // mirrors (which differ by `State` payload) it is reused directly // rather than duplicated. The source-order-then-reverse ordering // it encapsulates is load-bearing for suppression attribution. - // The returned child slice is only useful to `metrics_inner`, - // which seeds their cognitive nesting; `ops` just walks them. - push_children( + let pushed = push_children( &mut cursor, &node, Walk { @@ -377,6 +377,7 @@ pub(crate) fn ops_inner( }, &mut stack, ); + siblings.record(&node, depth, pushed.iter().rev().map(|(child, _)| *child)); } finalize::(&mut state_stack, usize::MAX); @@ -509,6 +510,32 @@ mod tests { ); } + /// A bareword key bills as the string it is (#1572): `-name` as one + /// operand with its sign, no `-` operator, and apart from the plain + /// `name` key. The expected vocabularies are the quoted twin's + /// (`'-name' => 1, 'name' => 2`, `$h{'-name'}`) with the quotes + /// dropped; before the fix `-name` billed `-` and folded into `name`. + /// The second fixture is the guard: a subtraction before a key and a + /// unary minus on a variable keep their `-`. + #[cfg(feature = "perl")] + #[test] + fn perl_bareword_keys_bill_as_written() { + check_ops( + LANG::Perl, + "my %h = (-name => 1, name => 2); my $v = $h{-name};", + "foo.pl", + &mut ["$", "()", ",", ";", "=", "=>", "my", "{}"], + &mut ["$h", "$v", "%h", "-name", "1", "2", "name"], + ); + check_ops( + LANG::Perl, + "my @a = (1 - size => 3, -$x);", + "foo.pl", + &mut ["()", ",", "-", ";", "=", "=>", "my", "$"], + &mut ["$x", "1", "3", "@a", "size"], + ); + } + #[cfg(feature = "python")] #[test] fn python_function_ops() { @@ -902,8 +929,10 @@ mod tests { } }", "foo.java", + // `public` and `static` are billed since #1565 &mut [ - "class", "{}", "void", "()", "[]", ",", ".", ";", "int", "=", "+", "/", + "public", "static", "class", "{}", "void", "()", "[]", ",", ".", ";", "int", "=", + "+", "/", ], &mut [ "Main", @@ -942,20 +971,11 @@ mod tests { }", "foo.java", // All 8 primitive-type keywords must appear as distinct operators. - // true/false appear as operands; `class` is billed since #1552. + // true/false appear as operands; `class` is billed since #1552 + // and `public` since #1565. &mut [ - "class", - "{}", - ";", - "=", - "byte", - "short", - "int", - "long", - "char", - "float", - "double", - "boolean_type", + "public", "class", "{}", ";", "=", "byte", "short", "int", "long", "char", "float", + "double", "boolean", ], &mut [ "Prims", "a", "b", "c", "d", "e", "f", "g", "h", "i", "1", "2", "3", "4", "'x'", @@ -1548,6 +1568,19 @@ mod tests { &["fn", "<>", "()", ":", "->", "impl", "+", "use", "{}"], [9, 10], ); + // A qualified path's `bracketed_type` encloses a type, `<>` with + // or without `as` (#1567). `<>` three times (two qualified paths + // and `Vec`), `()` three times, `::` and `;` twice; `as` still + // bills and the comparison `>` stays. Unfixed: [12, 19]. + check_generic_pair( + LANG::Rust, + "foo.rs", + "fn q() -> bool { as Default>::default(); ::g(); a > b }\n", + &[ + "fn", "()", "->", "bool", "{}", "<>", "u8", "as", "::", ";", ">", + ], + [11, 17], + ); } // expected: `<>` three times, `;` and `{}` twice each. @@ -1559,22 +1592,13 @@ mod tests { "foo.java", "class A { List> v; boolean m() { return a < b && c > d; } }\n", &[ - "class", - "<>", - ";", - "{}", - "boolean_type", - "()", - "return", - "<", - "&&", - ">", + "class", "<>", ";", "{}", "boolean", "()", "return", "<", "&&", ">", ], [10, 14], ); } - // expected: as Java, with `bool` for `boolean_type`. + // expected: as Java, with `bool` for `boolean`. #[cfg(feature = "csharp")] #[test] fn csharp_generic_angles_are_one_pair_1559() { @@ -1587,6 +1611,21 @@ mod tests { ], [10, 14], ); + // A function pointer type's brackets enclose its parameter and + // return types, `<>` with or without a calling convention (#1567). + // `;` three times; `{}`, `delegate`, `*`, `<>`, `int`, `,` and + // `void` twice. Unfixed: [16, 28], `<` and `>` three times each. + check_generic_pair( + LANG::Csharp, + "foo.cs", + "unsafe class C { delegate* p; delegate* unmanaged[Cdecl] q; \ + bool M() { return a < b && c > d; } }\n", + &[ + "unsafe", "class", "{}", "delegate", "*", "<>", "int", ",", "void", ";", "[]", + "bool", "()", "return", "<", "&&", ">", + ], + [17, 26], + ); } // expected: `<>` three times, `:` twice. TSX shares the getter macro @@ -1647,6 +1686,18 @@ mod tests { &["fun", "<>", "()", ":", "{}", "return", "<", "&&", ">"], [9, 12], ); + // A qualified `super` encloses the supertype it names (#1567). + // `()` three times (`A()`, `f()`, `.f()`), `:` and `{}` twice. + // Unfixed: [10, 16], `<` and `>` twice each. + check_generic_pair( + LANG::Kotlin, + "foo.kt", + "class C : A() { fun f(): Boolean { super.f(); return x < y && x > y } }\n", + &[ + "class", ":", "()", "{}", "fun", "<>", ".", "return", "<", "&&", ">", + ], + [11, 15], + ); } // expected: `<>` four times — Groovy spells a method's parameter @@ -1659,9 +1710,9 @@ mod tests { "foo.groovy", "class A { List> v; def boolean m() { return a < b && c > d } }\n", &[ - "class", "<>", ";", "{}", "def", "()", "return", "<", "&&", ">", + "class", "<>", ";", "{}", "def", "boolean", "()", "return", "<", "&&", ">", ], - [10, 14], + [11, 15], ); } @@ -1711,6 +1762,13 @@ mod tests { #[cfg(feature = "csharp")] #[test] fn csharp_misparsed_comparisons_stay_comparisons_1559() { + check_generic_pair( + LANG::Csharp, + "foo.cs", + "class A { void M() { F(Foo.a < b, c > d); } }\n", + &["()", ",", ".", ";", "<", "<>", ">", "class", "void", "{}"], + [10, 12], + ); check_generic_pair( LANG::Csharp, "foo.cs", diff --git a/src/spaces.rs b/src/spaces.rs index 7671ca52d..7385b468d 100644 --- a/src/spaces.rs +++ b/src/spaces.rs @@ -68,7 +68,7 @@ use crate::preproc::PreprocResults; use crate::checker::Checker; use crate::error::MetricsError; -use crate::node::{Ancestors, Cursor, Node}; +use crate::node::{Ancestors, Cursor, Node, SiblingCache}; use crate::suppression::{ Suppression, SuppressionKind, SuppressionScope, parse_marker as parse_suppression_marker, }; diff --git a/src/spaces/compute.rs b/src/spaces/compute.rs index 2b07944b5..51b6f3a10 100644 --- a/src/spaces/compute.rs +++ b/src/spaces/compute.rs @@ -428,7 +428,7 @@ fn compute_per_node<'a, T: MetricSuite>( T::NArgs::compute(node, code, ancestors, &mut last.metrics.nargs); } if selected.contains(Metric::Nexits) { - T::Exit::compute(node, code, &mut last.metrics.nexits); + T::Exit::compute(node, code, ancestors, &mut last.metrics.nexits); } if selected.contains(Metric::Abc) { T::Abc::compute(node, code, ancestors, &mut last.metrics.abc); @@ -755,6 +755,9 @@ pub(crate) fn metrics_inner( // index instead of through `Node::parent`, which `tree_sitter` // resolves by descending from the root (#1084). let mut chain: Vec> = Vec::new(); + // The children of each wide node on `chain`, so a predicate asking + // for a sibling in a flat `ERROR` does not scan it (#1584). + let mut siblings = SiblingCache::default(); let mut state_stack: Vec = Vec::new(); let mut last_level = 0; // Per-node cognitive nesting, inherited down the walk. Deliberately @@ -827,7 +830,7 @@ pub(crate) fn metrics_inner( // and `chain` is already correct here — the truncate above is // the only thing that touches it between the pop and this // point (#1100). - let ancestors = Ancestors::checked(&chain, &node); + let ancestors = Ancestors::checked(&chain, &node).with_siblings(&siblings); // Prune test-only subtrees before any per-metric work runs. // The hook is gated on `exclude_tests` so the default @@ -916,6 +919,7 @@ pub(crate) fn metrics_inner( }, &mut stack, ); + siblings.record(&node, depth, pushed.iter().rev().map(|(child, _)| *child)); if selected.contains(Metric::Cognitive) { propagate_nesting_to_children(&node, pushed, &mut nesting_map); diff --git a/src/wire.rs b/src/wire.rs index 5d69764c8..e7f866672 100644 --- a/src/wire.rs +++ b/src/wire.rs @@ -208,8 +208,8 @@ mod trend_wire_tests { assert!((from_json.ownership_top_share - row.ownership_top_share).abs() < 1e-12); // YAML round-trip. - let yaml = serde_yaml::to_string(&row).expect("serialize Vcs to YAML"); - let from_yaml: Vcs = serde_yaml::from_str(&yaml).expect("parse Vcs from YAML"); + let yaml = serde_saphyr::to_string(&row).expect("serialize Vcs to YAML"); + let from_yaml: Vcs = serde_saphyr::from_str(&yaml).expect("parse Vcs from YAML"); assert!(from_yaml.risk_score.is_nan() && from_yaml.burst.is_nan()); // CBOR round-trip. @@ -230,8 +230,9 @@ mod trend_wire_tests { vcs: sample_vcs(), }; - let yaml = serde_yaml::to_string(&point).expect("serialize VcsTrendPoint to YAML"); - let from_yaml: VcsTrendPoint = serde_yaml::from_str(&yaml).expect("parse point from YAML"); + let yaml = serde_saphyr::to_string(&point).expect("serialize VcsTrendPoint to YAML"); + let from_yaml: VcsTrendPoint = + serde_saphyr::from_str(&yaml).expect("parse point from YAML"); assert_eq!(from_yaml, point); let mut bytes = Vec::new(); @@ -264,8 +265,8 @@ mod trend_wire_tests { deltas: VcsTrendDeltas::default(), }; - let yaml = serde_yaml::to_string(&trend).expect("serialize VcsTrend to YAML"); - let from_yaml: VcsTrend = serde_yaml::from_str(&yaml).expect("parse VcsTrend from YAML"); + let yaml = serde_saphyr::to_string(&trend).expect("serialize VcsTrend to YAML"); + let from_yaml: VcsTrend = serde_saphyr::from_str(&yaml).expect("parse VcsTrend from YAML"); assert_eq!(from_yaml, trend); let mut bytes = Vec::new(); @@ -773,10 +774,10 @@ fn run() { #[test] fn yaml_round_trips() { check_func_space::(FIXTURE, "fixture.rs", |fs| { - let yaml = serde_yaml::to_string(&fs).expect("serialize to YAML"); - let back: FuncSpace = serde_yaml::from_str(&yaml).expect("parse wire from YAML"); + let yaml = serde_saphyr::to_string(&fs).expect("serialize to YAML"); + let back: FuncSpace = serde_saphyr::from_str(&yaml).expect("parse wire from YAML"); assert_eq!(back, fs.to_wire()); - assert_eq!(serde_yaml::to_string(&back).expect("re-serialize"), yaml); + assert_eq!(serde_saphyr::to_string(&back).expect("re-serialize"), yaml); }); } @@ -872,13 +873,13 @@ fn run() { "JSON null must deserialize back to NaN", ); - let yaml = serde_yaml::to_string(&mi).expect("YAML"); + let yaml = serde_saphyr::to_string(&mi).expect("YAML"); assert!( yaml.contains("original: null"), "non-finite must serialize to YAML null, got {yaml}", ); assert!( - serde_yaml::from_str::(&yaml) + serde_saphyr::from_str::(&yaml) .expect("parse") .original .is_nan() @@ -1118,7 +1119,7 @@ fn run() { ( [ serde_json::to_string(&at_limit).is_ok(), - serde_yaml::to_string(&at_limit).is_ok(), + serde_saphyr::to_string(&at_limit).is_ok(), toml::to_string(&at_limit).is_ok(), { let mut bytes = Vec::new(); @@ -1127,7 +1128,7 @@ fn run() { ], [ serde_json::to_string(&past_limit).is_ok(), - serde_yaml::to_string(&past_limit).is_ok(), + serde_saphyr::to_string(&past_limit).is_ok(), toml::to_string(&past_limit).is_ok(), { let mut bytes = Vec::new(); diff --git a/src/wire_ops_tests.rs b/src/wire_ops_tests.rs index af5a337dd..3ea2e06c8 100644 --- a/src/wire_ops_tests.rs +++ b/src/wire_ops_tests.rs @@ -87,8 +87,8 @@ fn borrowed_and_owned_ops_projections_serialize_alike() { "{lang:?} JSON must not depend on the projection" ); assert_eq!( - serde_yaml::to_string(&ops).expect("borrowed YAML"), - serde_yaml::to_string(&owned).expect("owned YAML"), + serde_saphyr::to_string(&ops).expect("borrowed YAML"), + serde_saphyr::to_string(&owned).expect("owned YAML"), "{lang:?} YAML must not depend on the projection" ); assert_eq!( diff --git a/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap b/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap index ca5892f24..10b604cdf 100644 --- a/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap +++ b/tests/output_formats/snapshots/output_formats__csv_test__csv_cpp_widget.snap @@ -3,8 +3,8 @@ source: tests/output_formats/csv_test.rs expression: out --- path,space_name,space_kind,start_line,end_line,cognitive.sum,cognitive.average,cognitive.min,cognitive.max,cyclomatic.sum,cyclomatic.average,cyclomatic.min,cyclomatic.max,cyclomatic.modified.sum,cyclomatic.modified.average,cyclomatic.modified.min,cyclomatic.modified.max,halstead.unique_operators,halstead.total_operators,halstead.unique_operands,halstead.total_operands,halstead.length,halstead.estimated_program_length,halstead.purity_ratio,halstead.vocabulary,halstead.volume,halstead.difficulty,halstead.level,halstead.effort,halstead.time,halstead.bugs,loc.sloc,loc.ploc,loc.lloc,loc.cloc,loc.blank,loc.sloc_average,loc.ploc_average,loc.lloc_average,loc.cloc_average,loc.blank_average,loc.sloc_min,loc.sloc_max,loc.cloc_min,loc.cloc_max,loc.ploc_min,loc.ploc_max,loc.lloc_min,loc.lloc_max,loc.blank_min,loc.blank_max,nom.functions,nom.closures,nom.functions_average,nom.closures_average,nom.total,nom.average,nom.functions_min,nom.functions_max,nom.closures_min,nom.closures_max,nargs.function_args,nargs.closure_args,nargs.function_args_average,nargs.closure_args_average,nargs.total,nargs.average,nargs.function_args_min,nargs.function_args_max,nargs.closure_args_min,nargs.closure_args_max,nexits.sum,nexits.average,nexits.min,nexits.max,tokens.sum,tokens.average,tokens.min,tokens.max,abc.assignments,abc.branches,abc.conditions,abc.magnitude,abc.assignments_average,abc.branches_average,abc.conditions_average,abc.assignments_min,abc.assignments_max,abc.branches_min,abc.branches_max,abc.conditions_min,abc.conditions_max,wmc.class_wmc_sum,wmc.interface_wmc_sum,wmc.total,npm.class_npm_sum,npm.interface_npm_sum,npm.class_methods,npm.interface_methods,npm.class_coa,npm.interface_coa,npm.total,npm.total_methods,npm.coa,npa.class_npa_sum,npa.interface_npa_sum,npa.class_attributes,npa.interface_attributes,npa.class_cda,npa.interface_cda,npa.total,npa.total_attributes,npa.cda,mi.original,mi.sei,mi.visual_studio -widget.cc,widget.cc,unit,1,10,0,0,0,0,5,2.5,1,1,5,2.5,1,1,10,20,6,9,29,48.72905595320056,1.680312274248295,16,116,7.5,0.13333333333333333,870,48.333333333333336,0.030377934896425568,10,9,2,0,1,2,1.8,0.4,0,0.2,1,10,0,0,1,9,1,2,0,1,2,0,0.4,0,2,0.4,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,7.6,0,12,1,0,0,1,0.2,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,107.82945249974337,80.37326368816134,63.05815935657507 -widget.cc,ns,namespace,2,10,0,0,0,0,4,2,1,1,4,2,1,1,10,20,6,9,29,48.72905595320056,1.680312274248295,16,116,7.5,0.13333333333333333,870,48.333333333333336,0.030377934896425568,9,9,2,0,0,2.25,2.25,0.5,0,0,1,9,0,0,1,9,1,2,0,0,2,0,0.5,0,2,0.5,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,9.5,5,12,1,0,0,1,0.25,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,109.76629285340017,83.06571380197117,64.19081453415215 -widget.cc,Widget,class,3,9,0,0,0,0,3,1.5,1,1,3,1.5,1,1,9,17,5,8,25,40.13896548741762,1.6055586194967049,14,95.1838730514401,7.2,0.1388888888888889,685.3238859703688,38.07354922057604,0.02591051177740593,7,7,2,0,0,2.3333333333333335,2.3333333333333335,0.6666666666666666,0,0,1,7,0,0,1,7,1,2,0,0,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,33,11,10,12,1,0,0,1,0.3333333333333333,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,115.09604084632235,90.65309494114345,67.3076262259195 -widget.cc,value,function,5,5,0,0,0,0,1,1,1,1,1,1,1,1,5,5,2,2,7,13.60964047443681,1.944234353490973,7,19.651484454403228,2.5,0.4,49.12871113600807,2.7293728408893374,0.004471319041702271,1,1,1,0,0,1,1,1,0,0,1,1,0,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,10,10,10,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,155.2836050228807,148.4278547652622,90.80912574437467 +widget.cc,widget.cc,unit,1,10,0,0,0,0,5,2.5,1,1,5,2.5,1,1,13,23,6,9,32,63.61549134016113,1.9879841043800353,19,135.93368043019473,9.75,0.10256410256410256,1325.3533841943986,73.63074356635548,0.040219179477315764,10,9,2,0,1,2,1.8,0.4,0,0.2,1,10,0,0,1,9,1,2,0,1,2,0,0.4,0,2,0.4,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,7.6,0,12,1,0,0,1,0.2,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,107.0048524543984,79.18361729202537,62.57593710783532 +widget.cc,ns,namespace,2,10,0,0,0,0,4,2,1,1,4,2,1,1,13,23,6,9,32,63.61549134016113,1.9879841043800353,19,135.93368043019473,9.75,0.10256410256410256,1325.3533841943986,73.63074356635548,0.040219179477315764,9,9,2,0,0,2.25,2.25,0.5,0,0,1,9,0,0,1,9,1,2,0,0,2,0,0.5,0,2,0.5,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,38,9.5,5,12,1,0,0,1,0.25,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,108.94169280805521,81.8760674058352,63.70859228541241 +widget.cc,Widget,class,3,9,0,0,0,0,3,1.5,1,1,3,1.5,1,1,12,20,5,8,28,54.62919048309068,1.9510425172532386,17,114.4489595550095,9.6,0.10416666666666667,1098.710011728091,61.03944509600506,0.03549229914069982,7,7,2,0,0,2.3333333333333335,2.3333333333333335,0.6666666666666666,0,0,1,7,0,0,1,7,1,2,0,0,2,0,0.6666666666666666,0,2,0.6666666666666666,0,1,0,0,1,0,0.5,0,1,0.5,0,1,0,0,1,0.5,0,1,33,11,10,12,1,0,0,1,0.3333333333333333,0,0,0,1,0,0,0,0,2,0,2,2,0,2,0,1,0,2,2,1,0,0,1,0,0,0,0,1,0,114.13758501653936,89.27033546860443,66.74712574066629 +widget.cc,value,function,5,5,0,0,0,0,1,1,1,1,1,1,1,1,6,6,2,2,8,17.509775004326936,2.188721875540867,8,24,3,0.3333333333333333,72,4,0.0057689982812296325,1,1,1,0,0,1,1,1,0,0,1,1,0,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,10,10,10,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,154.24412008219068,146.92819499625,90.20123981414659 widget.cc,set,function,6,6,0,0,0,0,1,1,1,1,1,1,1,1,6,6,3,4,10,20.264662506490403,2.0264662506490403,9,31.69925001442312,4,0.25,126.79700005769249,7.044277780982916,0.008413009854528818,1,1,1,0,0,1,1,1,0,0,1,1,0,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,1,0,1,0,1,1,1,1,0,0,0,0,0,0,12,12,12,12,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,152.79727628688778,144.8408406278255,89.35513233151332 diff --git a/tests/parity/abc_halstead_bracket_parity.rs b/tests/parity/abc_halstead_bracket_parity.rs index 8aa80317a..e556fc64a 100644 --- a/tests/parity/abc_halstead_bracket_parity.rs +++ b/tests/parity/abc_halstead_bracket_parity.rs @@ -49,12 +49,14 @@ use super::support::{metrics_space, ops_space}; /// for each rather than a `None` arm asserting they have no such /// construct, which would be false. What this pins is the policy, and a /// policy flip lands in a shared getter macro or a shared ABC arm, -/// which these seven rows already reach. Generic and template brackets -/// are the one family left out on purpose: Halstead bills them as a -/// single `<>` pair since #1559, which the `*_generic_angles_*` tests in -/// `src/ops.rs` pin per language. +/// which these six rows already reach. Angle brackets that enclose +/// types are the one family left out on purpose: Halstead bills them as +/// a single `<>` pair since #1559, which the `*_generic_angles_*` tests +/// in `src/ops.rs` pin per language. Kotlin's `super` joined that +/// family in #1567; its ABC half is +/// `kotlin_super_type_argument_is_not_a_condition`. // -// Two gates, one condition. The `mod` gate in `main.rs` names the seven +// Two gates, one condition. The `mod` gate in `main.rs` names the six // languages with a `Some` row and is the one that decides whether any of // this compiles: edit it when a row is added or removed. This gate and // the test's are the whole-roster union `check-test-lang-gates` derives @@ -147,22 +149,6 @@ fn fixture(lang: LANG) -> Option<(&'static str, &'static str, &'static [&'static .as_slice(), 22, ), - // A qualified super call disambiguates its supertype with the - // same two bare tokens. `override` bills as Kotlin's other - // modifiers do since #1558. - // - // expected N1 = 14: the header's `class`, `:`, `()` and `{}`, - // then `override`, `fun`, the parameter `()`, the return-type - // `:`, the body `{}`, `return`, `<`, `>`, `.` and the call's `()`. - LANG::Kotlin => ( - "class B : A() {\n override fun g(): Int { return super.g() }\n}\n", - "kt", - [ - "()", ".", ":", "<", ">", "class", "fun", "override", "return", "{}", - ] - .as_slice(), - 14, - ), // A filehandle readline. `` is deliberately not the // fixture: the grammar lexes it as one `standard_input` token, // so it has no brackets to classify and the row would assert @@ -262,7 +248,7 @@ fn a_non_comparison_bracket_is_a_halstead_operator_and_not_an_abc_condition() { } // Every language is feature-gated, so a build enabling none of the - // seven leaves a zero-iteration loop and a test that reports green + // six leaves a zero-iteration loop and a test that reports green // while asserting nothing. What keeps this from firing spuriously // is the `#[cfg(any(…))]` on this file's `mod` declaration in // `tests/parity/main.rs`, which names exactly the features whose diff --git a/tests/parity/cpp_mozcpp_parity.rs b/tests/parity/cpp_mozcpp_parity.rs index 73c9f03cb..b03cb66ca 100644 --- a/tests/parity/cpp_mozcpp_parity.rs +++ b/tests/parity/cpp_mozcpp_parity.rs @@ -200,3 +200,44 @@ fn cpp_and_mozcpp_agree_on_this() { "`this` must be billed as an operand: {cpp:?}" ); } + +#[cfg(all(feature = "cpp", feature = "mozcpp"))] +#[test] +fn cpp_and_mozcpp_agree_on_modifiers() { + // #1578 added the declaration modifiers to the operator arm of both + // `CppCode::get_op_type` and its `MozcppCode` clone. Mozcpp owns no + // file extension, so nothing else at the metric level exercises its + // copy. + // + // The fixture carries one of each wrapper the grammar puts a + // modifier in — `access_specifier`, `storage_class_specifier`, + // `virtual_specifier`, `type_qualifier` — beside a linkage + // specification whose `"C"` must stay an operand. + let source = "class C { public: static int a; virtual void f() const override final; \ + mutable int b; };\n\ + extern \"C\" inline constexpr int g() { return 1; }\n"; + + let cpp = metric_sums(LANG::Cpp, source, "cpp"); + let mozcpp = metric_sums(LANG::Mozcpp, source, "cpp"); + assert_eq!( + cpp, mozcpp, + "Cpp and Mozcpp must agree on modifier classification" + ); + + let get = |key: &str| metric(&cpp, key); + // Operators: `class`, `{` x2, `public`, `:`, `static`, `int` x3, + // `;` x5, `virtual`, `void`, `(` x2, `const`, `override`, `final`, + // `mutable`, `extern`, `inline`, `constexpr`, `return` -> N1 = 26. + // Before #1578 the ten modifiers billed nothing and this was 16. + assert_eq!( + get("halstead.operators"), + 26, + "each modifier must bill once as an operator: {cpp:?}" + ); + // Operands: `C`, `a`, `f`, `b`, `"C"`, `g`, `1` -> N2 = 7. + assert_eq!( + get("halstead.operands"), + 7, + "no modifier may bill as an operand: {cpp:?}" + ); +} diff --git a/tests/parity/main.rs b/tests/parity/main.rs index 7d2f25d69..20e1b0f1e 100644 --- a/tests/parity/main.rs +++ b/tests/parity/main.rs @@ -27,7 +27,6 @@ mod support; #[cfg(any( feature = "csharp", feature = "javascript", - feature = "kotlin", feature = "lua", feature = "mozjs", feature = "perl", diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index a737c2ff0..3c754c5af 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit a737c2ff02d8b9884700bf5e6d024c10e14ab9d1 +Subproject commit 3c754c5af769cd3e13303e2b1094250736612f5b diff --git a/utils/check-test-lang-gates-test.py b/utils/check-test-lang-gates-test.py index cb8a2915e..cf87fe994 100755 --- a/utils/check-test-lang-gates-test.py +++ b/utils/check-test-lang-gates-test.py @@ -796,6 +796,71 @@ def test_a_helper_called_from_a_narrower_helper_is_reported(self) -> None: [], ) + def test_a_language_bound_local_is_not_a_helper_call(self) -> None: + source = self.TWO_HOPS.replace( + " shared();\n }\n\n #[cfg(feature = \"python\")]", + " let shared = 1;\n assert_eq!(shared, 1);\n }\n\n #[cfg(feature = \"python\")]", + 1, + ) + self.assertEqual(_offending(source % 'feature = "python"'), []) + + def test_a_call_inherits_its_inner_guard(self) -> None: + source = self.TWO_HOPS.replace( + " shared();\n }\n\n #[cfg(feature = \"python\")]", + ' #[cfg(feature = "python")]\n shared();\n }\n\n #[cfg(feature = "python")]', + 1, + ) + self.assertEqual(_offending(source % 'feature = "python"'), []) + self.assertEqual( + _offending(self.TWO_HOPS % 'feature = "python"'), [("shared", ["rust"])] + ) + + def test_an_all_guard_keeps_a_matching_helper_gate(self) -> None: + source = ''' +#[cfg(test)] +mod tests { + #[cfg(all(feature = "python", feature = "rust"))] + fn shared() {} + #[test] + fn guarded_case() { + #[cfg(all(feature = "python", feature = "rust"))] + shared::(); + } +} +''' + self.assertEqual(_offending(source), []) + + def test_function_values_survive_a_local_in_another_scope(self) -> None: + source = ''' +#[cfg(test)] +mod tests { + #[cfg(feature = "python")] + fn shared() {} + #[cfg(feature = "python")] + #[test] + fn py() { check::(); shared(); } + #[cfg(feature = "rust")] + #[test] + fn rs() { + check::(); + consume(shared); + { let shared = 1; consume(shared); } + } +} +''' + self.assertEqual(_offending(source), [("shared", ["rust"])]) + mixed = source.replace( + "consume(shared);\n {", + '#[cfg(feature = "python")]\n shared();\n consume(shared);\n {', + 1, + ) + self.assertEqual(_offending(mixed), [("shared", ["rust"])]) + qualified = source.replace( + "consume(shared);\n { let shared = 1; consume(shared); }", + "let shared = 1;\n consume(self::shared);", + ) + self.assertEqual(_offending(qualified), [("shared", ["rust"])]) + def test_three_hops_through_a_helper_that_names_nothing(self) -> None: """The middle link names no language, so only its gate says python.""" source = """ @@ -1338,7 +1403,7 @@ def test_an_ungated_caller_strands_nothing_its_helper_never_needs(self) -> None: ) stray = _named(items, "stray") self.assertIsNone(gate.effective_predicate(items, stray)) - self.assertIn( + self.assertNotIn( stray.index, gate.callers_by_callee(items)[_named(items, "shared").index] ) self.assertEqual(gate.offenders(items, gate.resolve_needs(items)), []) @@ -2408,6 +2473,24 @@ def test_the_probe_set_comes_from_the_language_table(self) -> None: ) self.assertEqual(probes["--features rust"], frozenset({"rust"})) + def test_nested_file_modules_inherit_their_declaring_gate(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / "src" / "outer").mkdir(parents=True) + (root / "src" / "lib.rs").write_text("#[cfg(test)]\nmod outer;\n") + (root / "src" / "outer.rs").write_text( + '#[cfg(feature = "python")]\nmod suite;\n' + ) + (root / "src" / "outer" / "suite.rs").write_text( + "#[test]\nfn a_case() {}\n" + ) + sources = gate.rust_sources(root) + self.assertEqual( + gate.module_gates(root, sources)["src/outer/suite.rs"], + 'all(test, feature = "python")', + ) + self.assertIn("src/outer/suite.rs", gate.test_scope_files(root, sources)) + def test_membership_follows_the_gate(self) -> None: items = _scan( """ @@ -2496,6 +2579,68 @@ def git(*args: str) -> None: target.write_text("#[cfg(test)]\nmod tests {}\n") self.assertEqual(gate.compare_revisions("HEAD", root, table), []) + def test_a_root_mod_gate_that_narrowed_is_reported(self) -> None: + # The gate sits on the `mod` line in the crate root and the test + # file says nothing of it, so per-item membership alone cannot + # see the narrowing -- #1567 dropped `kotlin` from + # `tests/parity/main.rs` this way and `--compare` reported OK. + wide_root = ( + '#[cfg(any(feature = "python", feature = "rust"))]\n' + "mod suite;\n" + "mod ungated;\n" + ) + suite = """#[test] +fn a_case() { + for lang in LANG::into_enum_iter().filter(LANG::is_enabled) { + let _ = (LANG::Python, LANG::Rust, lang); + } +} +""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + crate = root / "tests" / "parity" + crate.mkdir(parents=True) + (root / "big-code-analysis-ast" / "src").mkdir(parents=True) + (root / "big-code-analysis-ast" / "src" / "langs.rs").write_text( + LANGS_FIXTURE + ) + main_rs = crate / "main.rs" + main_rs.write_text(wide_root) + (crate / "suite.rs").write_text(suite) + (crate / "ungated.rs").write_text(suite) + + env = { + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t", + "PATH": os.environ.get("PATH", ""), + "HOME": str(root), + } + for args in (["init", "-q"], ["add", "-A"], ["commit", "-q", "-m", "b"]): + subprocess.run( + ["git", *args], cwd=root, check=True, capture_output=True, env=env + ) + + table = gate.language_table(LANGS_FIXTURE) + self.assertEqual(gate.compare_revisions("HEAD", root, table), []) + + # Only the `mod` line narrows. `suite.rs` is byte-identical, + # and `ungated.rs`, loaded by an ungated `mod`, is the + # control: it must not be reported. + main_rs.write_text( + wide_root.replace( + 'any(feature = "python", feature = "rust")', 'feature = "python"' + ) + ) + self.assertEqual( + [ + (name, sorted(lost)) + for name, lost in gate.compare_revisions("HEAD", root, table) + ], + [("tests/parity/suite.rs::a_case", ["--features rust"])], + ) + class RepositoryTest(unittest.TestCase): def setUp(self) -> None: diff --git a/utils/check-test-lang-gates.py b/utils/check-test-lang-gates.py index 69ef86e21..25d4c5de1 100755 --- a/utils/check-test-lang-gates.py +++ b/utils/check-test-lang-gates.py @@ -838,6 +838,9 @@ class Item: #: the dataclass stays comparable and printable. index: int in_test_scope: bool + #: Exact inner predicates at each call/reference, retained for implication checks. + call_guards: tuple[tuple[str, str], ...] = () + reference_guards: tuple[tuple[str, str], ...] = () def describe(self) -> str: return f"{self.path}:{self.line} {self.kind} {self.name}" @@ -860,16 +863,27 @@ def qualified_name(items: list[Item], item: Item) -> str: def membership( - per_file: dict[str, list[Item]], probes: dict[str, frozenset[str]] + per_file: dict[str, list[Item]], + probes: dict[str, frozenset[str]], + file_gates: dict[str, str] | None = None, ) -> dict[str, frozenset[str]]: - """Which probe builds compile each test, keyed by qualified name.""" + """Which probe builds compile each test, keyed by qualified name. + + ``file_gates`` (see `module_gates`) conjoins the ``cfg`` of the `mod` + line that loads each file onto every test in it. + """ languages = frozenset().union(*probes.values()) if probes else frozenset() result: dict[str, frozenset[str]] = {} - for items in per_file.values(): + for path, items in per_file.items(): + file_gate = (file_gates or {}).get(path) for item in items: if not (item.is_test_fn and item.in_test_scope): continue predicate = effective_predicate(items, item) + if file_gate is not None: + predicate = ( + file_gate if predicate is None else f"all({file_gate}, {predicate})" + ) if predicate is None: result[qualified_name(items, item)] = frozenset(probes) continue @@ -1007,6 +1021,35 @@ def _inner_cfg_extents( return extents +def _shadowed_references(body: str) -> set[int]: + """Offsets naming simple locals, respecting block scope and initializer use.""" + tokens = re.compile( + r"\blet\s+(?:mut\s+)?([a-z_][A-Za-z0-9_]*)|[{};]|\b[A-Za-z_][A-Za-z0-9_]*\b" + ) + scopes: list[set[str]] = [set()] + pending: list[set[str]] = [set()] + shadowed: set[int] = set() + for token in tokens.finditer(body): + name = token.group(1) + if name is not None: + shadowed.add(token.start(1)) + pending[-1].add(name) + elif token.group() == "{": + scopes.append(set()) + pending.append(set()) + elif token.group() == "}" and len(scopes) > 1: + scopes.pop() + pending.pop() + elif token.group() == ";": + scopes[-1].update(pending[-1]) + pending[-1].clear() + elif body[max(0, token.start() - 2):token.start()] == "::": + continue + elif any(token.group() in scope for scope in scopes): + shadowed.add(token.start()) + return shadowed + + def scan_source( source: str, path: str, @@ -1355,12 +1398,18 @@ def admits(disabled: frozenset[str], gates: list[str] = guarded[n]) -> bool: unguarded_references: set[str] = set() guarded_calls: list[tuple[str, frozenset[str]]] = [] guarded_references: list[tuple[str, frozenset[str]]] = [] - for pattern, unguarded_set, guarded_list in ( - (CALL_RE, unguarded_calls, guarded_calls), - (REFERENCE_RE, unguarded_references, guarded_references), + call_guards: list[tuple[str, str]] = [] + reference_guards: list[tuple[str, str]] = [] + shadowed = _shadowed_references(body) + for pattern, unguarded_set, guarded_list, sites in ( + (CALL_RE, unguarded_calls, guarded_calls, call_guards), + (REFERENCE_RE, unguarded_references, guarded_references, reference_guards), ): for hit in pattern.finditer(body): + if hit.start(1) in shadowed: + continue n = index + bisect.bisect_right(line_starts, hit.start()) - 1 + sites.append((hit.group(1), "all(" + ", ".join(guarded.get(n, ())) + ")")) languages = attribution[innermost[n]] if n in guarded else None if languages is None: unguarded_set.add(hit.group(1)) @@ -1389,15 +1438,17 @@ def admits(disabled: frozenset[str], gates: list[str] = guarded[n]) -> bool: direct=frozenset(direct), hardcoded=frozenset(hardcoded), hand_written=_hand_written_features(raw, attr_line), - calls=frozenset(m.group(1) for m in CALL_RE.finditer(body)), + calls=frozenset(name for name, _ in call_guards), unguarded_calls=frozenset(unguarded_calls), guarded_calls=tuple(guarded_calls), - references=frozenset(m.group(1) for m in REFERENCE_RE.finditer(body)), + references=frozenset(name for name, _ in reference_guards), unguarded_references=frozenset(unguarded_references), guarded_references=tuple(guarded_references), parent_index=parent_index, index=len(items), in_test_scope=in_test_scope, + call_guards=tuple(call_guards), + reference_guards=tuple(reference_guards), ) items.append(item) depth += line.count("{") - line.count("}") @@ -1445,53 +1496,113 @@ def rust_sources(root: pathlib.Path) -> list[pathlib.Path]: PATH_ATTR_RE = re.compile(r'#\[\s*path\s*=\s*"([^"]+)"\s*\]') -def test_scope_files(root: pathlib.Path, sources: list[pathlib.Path]) -> set[str]: - """Files that are wholly test modules by their declaration. +def _module_declarations( + root: pathlib.Path, sources: list[pathlib.Path] +) -> list[tuple[pathlib.Path, list[str], pathlib.Path]]: + """Every out-of-line ``mod x;`` as (declaring file, attributes, file). - `src/spaces_tests.rs` and its siblings hold nothing but tests, yet - carry no `#[cfg(test)]` of their own: the marker is on the - `#[cfg(test)] mod spaces_tests;` line in `lib.rs`. Reading that - declaration is what puts their 94 top-level tests in scope -- - without it they are taken for production code and left ungated, - which is 79 panics on a single-language leg. + Only declarations whose module file exists are returned. The + attributes are the outer ones stacked on the declaration; a file's + leading ``#![…]`` run is dropped, since it belongs to the file and + not to the first `mod` that happens to follow it. """ - scoped: set[str] = set() + found: list[tuple[pathlib.Path, list[str], pathlib.Path]] = [] for relative in sources: text = (root / relative).read_text(encoding="utf-8") lines = mask_dead(text).split("\n") raw = text.split("\n") index = 0 while index < len(lines): - if not lines[index].strip().startswith(("#[", "#![")): - index += 1 - continue - attrs, after = _join_attributes(lines, raw, index) + if lines[index].strip().startswith(("#[", "#![")): + attrs, after = _join_attributes(lines, raw, index) + attrs = [a for a in attrs if not a.startswith("#![")] + else: + attrs, after = [], index index = max(after, index + 1) if after >= len(lines): break declared = MOD_DECL_RE.match(lines[after]) if declared is None: continue - predicates = [cfg_predicate(a) for a in attrs] - if not any(p and predicate_requires_test(p) for p in predicates): - continue + index = after + 1 # `#[path = "…"]` renames the file the module comes from: # `src/spaces.rs` declares `#[path = "spaces_tests.rs"] mod # tests;`, so the name says `tests` and the file does not. override = PATH_ATTR_RE.search(" ".join(attrs)) name = declared.group(1) + module_dir = ( + relative.parent + if relative.name in {"lib.rs", "main.rs", "mod.rs"} + else relative.with_suffix("") + ) candidates = ( [relative.parent / override.group(1)] if override else [ - relative.parent / f"{name}.rs", - relative.parent / name / "mod.rs", + module_dir / f"{name}.rs", + module_dir / name / "mod.rs", ] ) - for candidate in candidates: - if (root / candidate).is_file(): - scoped.add(candidate.as_posix()) - return scoped + found.extend( + (relative, attrs, candidate) + for candidate in candidates + if (root / candidate).is_file() + ) + return found + + +def test_scope_files(root: pathlib.Path, sources: list[pathlib.Path]) -> set[str]: + """Files that are wholly test modules by their declaration. + + `src/spaces_tests.rs` and its siblings hold nothing but tests, yet + carry no `#[cfg(test)]` of their own: the marker is on the + `#[cfg(test)] mod spaces_tests;` line in `lib.rs`. Reading that + declaration is what puts their 94 top-level tests in scope -- + without it they are taken for production code and left ungated, + which is 79 panics on a single-language leg. + """ + return { + path for path, predicate in module_gates(root, sources).items() + if predicate_requires_test(predicate) + } + + +def module_gates(root: pathlib.Path, sources: list[pathlib.Path]) -> dict[str, str]: + """Each file's ``cfg`` as inherited from the `mod` lines that load it. + + A `#[cfg(any(feature = …))] mod suite;` in `tests/parity/main.rs` + gates every test in `tests/parity/suite.rs`, but nothing in that + file says so. `--compare` diffs per-test membership, so a narrowed + `mod` gate dropped a whole suite from a single-language leg with + nothing reporting it -- #1567's `kotlin` drop went unseen that way. + Gates conjoin down a chain of declarations; a file no gated `mod` + reaches is absent from the result and keeps its own membership. + """ + declared: dict[str, tuple[str, list[str]]] = {} + for parent, attrs, module in _module_declarations(root, sources): + predicates = [p for p in (cfg_predicate(a) for a in attrs) if p is not None] + declared[module.as_posix()] = (parent.as_posix(), predicates) + + resolved: dict[str, list[str]] = {} + + def chain(path: str, seen: frozenset[str]) -> list[str]: + if path in resolved: + return resolved[path] + if path not in declared or path in seen: + return [] + parent, own = declared[path] + parts = [*chain(parent, seen | {path}), *own] + resolved[path] = parts + return parts + + gates: dict[str, str] = {} + for path in declared: + parts = chain(path, frozenset()) + if parts: + gates[path] = ( + parts[0] if len(parts) == 1 else "all(" + ", ".join(parts) + ")" + ) + return gates def scan_tree(root: pathlib.Path, table: dict[str, str]) -> dict[str, list[Item]]: @@ -1742,6 +1853,9 @@ def reachable(item: Item, generic_too: bool) -> set[str]: continue if not _uses(item, items[callee_index], strict=False): continue + unguarded = item.unguarded_references + if name not in unguarded: + continue addition = needs[item.index] - needs[callee_index] if addition: needs[callee_index] |= addition @@ -2139,17 +2253,13 @@ def stranded_by_callers( ``predicate`` is the helper's own effective gate. An ungated helper never gets here: the union check has already reported it. - Only a build enabling something in ``required`` counts. The caller - link is the broad one, so a local that merely shares the helper's - name links an ungated caller to it, and that caller is in every - build; without the intersection the helper reads as stranded in all - of them, and the `any(required)` marker `--fix` writes can never - clear it. A real caller's languages are in ``required`` already, - since needs propagate over the same links, so the `any(…)` marker - `--fix` writes clears whatever this reports — when it can replace the - helper's gate. A gate `--fix` must stack on instead (an `all(…)`, a - `not(…)`, a non-language atom) can still conjoin it away; the fix - loop then reports non-convergence rather than looping. + Each edge conjoins the caller's gate with the inner predicates on + that particular use. A guarded call is absent when its guard is + false, even though the containing function still compiles. Only a + build enabling something in ``required`` counts, so reports remain + expressible by the derived helper marker. A gate `--fix` must stack + on instead (an `all(…)`, a `not(…)`, a non-language atom) can still + conjoin it away; the fix loop then reports non-convergence. A helper whose body names no language has an empty ``required`` and is never reported, even when its own gate excludes a caller's build @@ -2158,12 +2268,17 @@ def stranded_by_callers( """ stranded: set[str] = set() for index in callers.get(item.index, []): - caller_gate = effective_predicate(items, items[index]) - for disabled in _caller_builds(caller_gate): - if ( - caller_gate is None or evaluate_predicate(caller_gate, disabled) - ) and not evaluate_predicate(predicate, disabled): - stranded |= (KNOWN_FEATURES - disabled) & required + caller = items[index] + caller_gate = effective_predicate(items, caller) or "all()" + sites = caller.reference_guards + guards = {guard for name, guard in sites if name == item.name} or {"all()"} + for guard in guards: + call_gate = f"all({caller_gate}, {guard})" + for disabled in _caller_builds(call_gate): + if evaluate_predicate(call_gate, disabled) and not evaluate_predicate( + predicate, disabled + ): + stranded |= (KNOWN_FEATURES - disabled) & required return frozenset(stranded) @@ -2372,7 +2487,9 @@ def apply_fixes( # --------------------------------------------------------------------------- -def scan_revision(ref: str, root: pathlib.Path) -> dict[str, list[Item]]: +def scan_revision( + ref: str, root: pathlib.Path +) -> tuple[dict[str, list[Item]], dict[str, str]]: """Scan the tree as of ``ref``, extracted to a scratch directory. `git archive` rather than a worktree: it touches nothing in the @@ -2398,7 +2515,9 @@ def scan_revision(ref: str, root: pathlib.Path) -> dict[str, list[Item]]: table = language_table( (checkout / LANGS_RS.relative_to(REPO_ROOT)).read_text(encoding="utf-8") ) - return scan_tree(checkout, table) + return scan_tree(checkout, table), module_gates( + checkout, rust_sources(checkout) + ) finally: shutil.rmtree(checkout, ignore_errors=True) @@ -2413,7 +2532,7 @@ def compare_revisions( both wrong, only the previous revision says so. """ probes = probe_builds(table) - reference_tree = scan_revision(ref, root) + reference_tree, reference_gates = scan_revision(ref, root) # The same guard `main` applies to the working tree. A reference with # no tests in it makes every name miss the `after` side, so nothing # is ever reported and the check prints OK having compared nothing — @@ -2429,8 +2548,10 @@ def compare_revisions( "against. A scan root is missing from that revision, or is " "excluded from `git archive` by `.gitattributes`." ) - before = membership(reference_tree, probes) - after = membership(scan_tree(root, table), probes) + before = membership(reference_tree, probes, reference_gates) + after = membership( + scan_tree(root, table), probes, module_gates(root, rust_sources(root)) + ) lost: list[tuple[str, frozenset[str]]] = [] for name, was in sorted(before.items()): # Only a test that is still here. One that was deleted or renamed