From 4e560d76f76adb7d8b1045cdd7877d8f406c44d8 Mon Sep 17 00:00:00 2001 From: Karib0u Date: Sun, 2 Aug 2026 22:17:02 +0200 Subject: [PATCH] chore: cut local `cargo test` from 6m43s to 20s, fix two loop defects `cargo test --workspace` was 6 m 43 s at 131% CPU. Three files were 383 s of the 402 s: cross_product_matrix (197 s, four tests each reloading the 1,042-rule corpus over 11 shapes), aarch64_elf_fuzz (166 s, 900 mutants), and dotnet_dnfile_fuzz (20 s, 2,400 mutants). The other 21 test files ran in ~18 s combined. All three are acceptance and robustness gates that an ordinary edit does not move, so they were paying inner-loop cost for pre-merge value. `#[ignore]` them and have CI opt back in with `--include-ignored`. Local `cargo test --workspace` is now 20 s; the CI command runs all 6 with 0 ignored and passes. The MSRV job deliberately keeps the plain command: it asks whether the workspace compiles and passes on the Rust floor, the ignored tests are still compiled there, and re-running the matrix on a second runner tests nothing about the Rust version. Two defects found while measuring: difftest.py defaulted `--capa-cli` to `target/debug/capa`, which is not the binary's name, so a run that omitted the flag could not find it. Worse, on a tree still holding a stale artifact under the old name it would difftest that instead -- and since the harness caches capa-x's side by binary contents, the result would look clean and self-consistent while measuring the wrong build. Point the default at `capa-x`. The outer loop had no `corpus-outer.expected.json`, so difftest fell through the baseline branch to `if failures: return 1`. With 91 known divergences the pre-merge gate AGENTS.md documents could never pass, only be read by hand. Record the baseline after reviewing the run: 200 samples, 98.55% rule-level agreement, 161/200 identical, 91 divergences, 0 errored -- matching the figure the README and CHANGELOG already publish, so this pins the released state rather than freezing a regression. The loop now exits 0 and reports regressions against known diffs. Docs: split CONTRIBUTING's pre-PR block into "before pushing" and "before merge or release", and give both loop tables measured costs that distinguish warm from post-rebuild -- the outer loop is ~2 min after a rebuild but ~2 s without one, and that gap is the difference between the loops reading as cheap and reading as punitive. --- .github/workflows/ci.yml | 12 +- AGENTS.md | 20 +- CHANGELOG.md | 23 +- CONTRIBUTING.md | 49 +- capa-x/tests/aarch64_elf_fuzz.rs | 6 + capa-x/tests/cross_product_matrix.rs | 10 + capa-x/tests/dotnet_dnfile_fuzz.rs | 5 + scripts/corpus-outer.expected.json | 935 +++++++++++++++++++++++++++ scripts/difftest.py | 2 +- 9 files changed, 1046 insertions(+), 16 deletions(-) create mode 100644 scripts/corpus-outer.expected.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c474c9..c550a8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,8 +62,13 @@ jobs: - name: cargo clippy run: cargo clippy --workspace --all-targets -- -D warnings + # `--include-ignored`: the slow acceptance/robustness gates + # (cross_product_matrix, aarch64_elf_fuzz, dotnet_dnfile_fuzz -- ~383 s + # of the suite's ~402 s) are `#[ignore]`d so a local `cargo test` stays + # a ~20 s inner loop. CI is where they must actually run, so it opts + # back in. Dropping this flag silently removes them from coverage. - name: cargo test - run: cargo test --workspace + run: cargo test --workspace -- --include-ignored # Gate J2 (docs/BENCHMARKS.md): `--jobs N` output must be byte-identical to # `--jobs 1`. `cargo test` above already runs the library-level half @@ -125,6 +130,11 @@ jobs: # Build and test, not just check: `cargo check` would miss anything that # only fails at codegen or link time. + # Deliberately without `--include-ignored`, unlike the `build` job: this + # job asks "does the workspace still compile and pass on the MSRV", and + # the ignored gates are still *compiled* here (that is the codegen/link + # coverage MSRV cares about). Re-running ~383 s of acceptance matrix on + # a second runner would not test anything about the Rust version. - name: cargo test on the declared MSRV run: cargo test --workspace --locked diff --git a/AGENTS.md b/AGENTS.md index c3dcca6..439c1ef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,14 +17,24 @@ results. ## Feedback loops -| Loop | Command | When | -|---|---|---| -| Inner | `cargo test -p capa-x --test features_parity` | Every change | -| Mid | `python3 scripts/difftest.py --mode full --samples scripts/corpus-smoke.txt --capa-cli target/release/capa-x --jobs 6` | Every commit | -| Outer | The mid command with `scripts/corpus-outer.txt` | Before merge or release | +| Loop | Command | When | Cost | +|---|---|---|---| +| Inner | `cargo test -p capa-x --test features_parity` | Every change | ~7 s | +| Mid | `python3 scripts/difftest.py --mode full --samples scripts/corpus-smoke.txt --capa-cli target/release/capa-x --jobs 6` | Every commit | ~40 s | +| Outer | The mid command with `scripts/corpus-outer.txt` | Before merge or release | ~5 min | + +Use the cheapest loop that can observe the change. `cargo test --workspace` +(~20 s) is the broader local check; the slow acceptance and robustness gates +are `#[ignore]`d and belong to CI, which runs them with `--include-ignored`. +Run them locally with the same flag before merge, not on every edit. `--jobs 1` is the semantic baseline. Any byte difference across job counts is a bug. Never carry a measurement forward from a note or cached report. +Both difftest corpora carry a per-sample baseline (`.expected.json`), +so a run reports regressions against known diffs rather than the raw presence +of a diff. Re-record with `--write-expected` only for a deliberate change, in +the same commit, with the reason stated. + See [`CONTRIBUTING.md`](CONTRIBUTING.md) for setup, complete checks, and the pull request checklist. diff --git a/CHANGELOG.md b/CHANGELOG.md index eb25bd4..e3251b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,28 @@ tracked per class in [KNOWN_DIVERGENCES.md](KNOWN_DIVERGENCES.md). ## [Unreleased] -No unreleased changes. +### Fixed + +- `scripts/difftest.py` defaulted `--capa-cli` to `target/debug/capa`, which + is not the binary's name. A run that omitted the flag could not find it, and + on a tree that still held a stale artifact under the old name it would + difftest that instead -- and since the harness caches capa-x's side by + binary contents, the result would look clean and self-consistent while + measuring the wrong build. The default is now `target/debug/capa-x`. + +### Changed + +- The slow acceptance and robustness gates + (`capa-x/tests/cross_product_matrix.rs`, `aarch64_elf_fuzz.rs`, + `dotnet_dnfile_fuzz.rs`) are `#[ignore]`d: ~383 s of a ~402 s suite, none of + which an ordinary edit moves. `cargo test --workspace` is now ~20 s + locally. CI runs the full set with `--include-ignored`, so coverage is + unchanged. +- `scripts/corpus-outer.expected.json` records the 200-sample outer corpus + baseline (98.55% rule-level agreement, 161/200 identical, 91 divergences, + 0 errors). Without it the outer loop had no baseline to resolve and exited + nonzero on every run, so the pre-merge gate `AGENTS.md` documents could not + pass; it now reports regressions against known diffs. ## [1.0.0] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c9a5fd8..a821ebb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,29 +43,62 @@ Python reference in `.venv`, and verifies every pin against `PINNED.md`. Use the cheapest loop that can observe the change: -| Loop | Command | When | -|---|---|---| -| Inner | `cargo test -p capa-x --test features_parity` | Every change | -| Mid | `python3 scripts/difftest.py --mode full --samples scripts/corpus-smoke.txt --capa-cli target/release/capa-x --jobs 6` | Every commit | -| Outer | The mid command with `scripts/corpus-outer.txt` | Before merge or release | +| Loop | Command | When | Cost | +|---|---|---|---| +| Inner | `cargo test -p capa-x --test features_parity` | Every change | ~7 s | +| Mid | `python3 scripts/difftest.py --mode full --samples scripts/corpus-smoke.txt --capa-cli target/release/capa-x --jobs 6` | Every commit | ~40 s | +| Outer | The mid command with `scripts/corpus-outer.txt` | Before merge or release | ~5 min | The inner loop is the transcribed upstream feature contract. The mid loop is the smoke regression guard. The outer loop measures rule-level agreement and the extra-rule count across the full corpus. +The difftest costs above are what you pay *after a code change*, on a 10-core +M1 Max; treat them as an order of magnitude, not a benchmark. Both sides are +cached under `.cache/`: the Python reference by sample hash, capa-x's own +output by binary contents. So a rebuilt binary invalidates only the capa-x +side (mid ~40 s, outer ~5 min), and re-running either loop without rebuilding +is 1-2 s. `.cache/` is disposable, but deleting it costs a one-time +re-analysis of the corpus under the pinned Python reference, which is slow +(minutes to hours) -- prefer `--no-rust-cache` when you want to distrust a +capa-x result, rather than clearing the whole cache. + `--jobs 1` is the semantic baseline for analysis output. Harness-level jobs only parallelize independent samples. Any byte difference between analysis with `--jobs 1` and another job count is a bug. -Before submitting a pull request, run: +### Before pushing ```bash -cargo build && cargo test -cargo fmt --check +cargo fmt --all -- --check cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace +``` + +`cargo test --workspace` is about 20 seconds. The slow acceptance and +robustness gates are `#[ignore]`d so this stays an inner loop: the +cross-product matrix and the two byte-flip fuzz suites are ~383 s of a ~402 s +suite between them, and an ordinary edit does not move them. CI runs the full +set with `--include-ignored` on all three platforms, so pushing without having +run them locally is expected, not a shortcut. + +### Before merge or release + +Run what CI cannot run cheaply on every push, plus the ignored gates: + +```bash +cargo test --workspace -- --include-ignored scripts/check_env.sh +python3 scripts/difftest.py --mode full --samples scripts/corpus-outer.txt \ + --capa-cli target/release/capa-x --jobs 6 ``` +Both difftest corpora have a recorded per-sample baseline +(`.expected.json`), so the exit status reports *regressions* against +known diffs rather than the raw presence of any diff. A deliberate change to +what matches is re-recorded with `--write-expected`, in the same commit, with +the reason in the pull request. + Use the differential harness for behavioral changes: ```bash diff --git a/capa-x/tests/aarch64_elf_fuzz.rs b/capa-x/tests/aarch64_elf_fuzz.rs index c908d7a..764b570 100644 --- a/capa-x/tests/aarch64_elf_fuzz.rs +++ b/capa-x/tests/aarch64_elf_fuzz.rs @@ -53,7 +53,13 @@ fn fixture_bytes(name: &str) -> Vec { std::fs::read(&path).unwrap_or_else(|e| panic!("reading {}: {e}", path.display())) } +// 900 mutants through the full ELF pipeline, single-threaded: ~166 s, the +// second largest cost in `cargo test --workspace`. It guards against rare +// panics on malformed input, not against the behavior an ordinary edit moves, +// so it is a pre-merge gate rather than an inner-loop test. CI runs it via +// `cargo test --workspace -- --include-ignored`. #[test] +#[ignore = "slow robustness gate; run with --include-ignored"] fn mutated_aarch64_elf_samples_never_panic() { let default_hook = panic::take_hook(); panic::set_hook(Box::new(|_| {})); diff --git a/capa-x/tests/cross_product_matrix.rs b/capa-x/tests/cross_product_matrix.rs index 8fe27ea..6f6b424 100644 --- a/capa-x/tests/cross_product_matrix.rs +++ b/capa-x/tests/cross_product_matrix.rs @@ -23,6 +23,12 @@ //! separate gate -- this only proves the Rust-side shape is //! self-consistent, matching `capa-x/tests/schema_roundtrip.rs`'s own //! stated scope. +//! +//! Every test here is `#[ignore]`d: the four of them together are ~197 s, the +//! single largest cost in `cargo test --workspace`, because each reloads the +//! 1,042-rule corpus and runs all 11 shapes (cell 1 also at four job counts). +//! This is a pre-merge acceptance matrix, not a test an ordinary edit moves. +//! CI runs it via `cargo test --workspace -- --include-ignored`. #![allow(clippy::unwrap_used, clippy::expect_used)] @@ -181,6 +187,7 @@ fn normalized_json(doc: &ResultDocument) -> serde_json::Value { /// byte-identical after timestamp normalization, matching AGENTS.md's /// blanket rule for every backend. #[test] +#[ignore = "slow acceptance gate; run with --include-ignored"] fn jobs_1_matches_default_for_every_shape() { let rules = ruleset(); let mut failures = Vec::new(); @@ -226,6 +233,7 @@ fn jobs_1_matches_default_for_every_shape() { /// thread count (e.g. an unstable sort, an uninitialized-memory read, a /// HashMap iteration order leaking into output). #[test] +#[ignore = "slow acceptance gate; run with --include-ignored"] fn repeated_output_is_identical_for_every_shape() { let rules = ruleset(); let mut failures = Vec::new(); @@ -260,6 +268,7 @@ fn repeated_output_is_identical_for_every_shape() { /// produced document back into `ResultDocument` and re-serializes to the /// same structure. See the module doc for how this differs from J14. #[test] +#[ignore = "slow acceptance gate; run with --include-ignored"] fn result_document_round_trips_for_every_shape() { let rules = ruleset(); let mut failures = Vec::new(); @@ -340,6 +349,7 @@ impl Rng { const MUTANTS_PER_SHAPE: usize = 15; #[test] +#[ignore = "slow acceptance gate; run with --include-ignored"] fn malformed_input_never_panics_through_the_full_pipeline() { let rules = ruleset(); let default_hook = panic::take_hook(); diff --git a/capa-x/tests/dotnet_dnfile_fuzz.rs b/capa-x/tests/dotnet_dnfile_fuzz.rs index 656fe67..a3a5838 100644 --- a/capa-x/tests/dotnet_dnfile_fuzz.rs +++ b/capa-x/tests/dotnet_dnfile_fuzz.rs @@ -60,7 +60,12 @@ fn corpus_dir() -> PathBuf { .join("tests/testfiles/dotnet") } +// 2,400 mutants through `dotnet::load`, single-threaded: ~20 s. Same shape as +// `aarch64_elf_fuzz.rs` -- a malformed-input panic guard, not a test an +// ordinary edit moves. CI runs it via +// `cargo test --workspace -- --include-ignored`. #[test] +#[ignore = "slow robustness gate; run with --include-ignored"] fn mutated_dotnet_samples_never_panic() { // The vendored fork's own panic hook still prints backtraces on our // caught panics; keep test output readable and restore it afterward. diff --git a/scripts/corpus-outer.expected.json b/scripts/corpus-outer.expected.json new file mode 100644 index 0000000..709547e --- /dev/null +++ b/scripts/corpus-outer.expected.json @@ -0,0 +1,935 @@ +{ + "mode": "full", + "samples": { + "009c2377b67997b0da1579f4bbc822c1.exe_": { + "diffs": 0, + "details": [] + }, + "021f49678cd633dc8cf99c61b3af3dda.exe_": { + "diffs": 0, + "details": [] + }, + "03b236b23b1ec37c663527c1f53af3fe.dll_": { + "diffs": 0, + "details": [] + }, + "03bb32d43885e83bc56c0b2bcad6f0c5ea40402763b7057056c654990022471b.dll_": { + "diffs": 0, + "details": [] + }, + "055da8e6ccfe5a9380231ea04b850e18.elf_": { + "diffs": 0, + "details": [] + }, + "0596c4ea5aa8def47f22c85d75aaca95.exe_": { + "diffs": 0, + "details": [] + }, + "0731679c5f99e8ee65d8b29a3cabfc6b.exe_": { + "diffs": 0, + "details": [] + }, + "0761142efbda6c4b1e801223de723578.dll_": { + "diffs": 0, + "details": [] + }, + "07f7846bbcda782e5639292ad93907eb.exe_": { + "diffs": 0, + "details": [] + }, + "09bf850be5da44a1c3629a1f62813a83.dll_": { + "diffs": 0, + "details": [] + }, + "0a30182ff3a6b67beb0f2cda9d0de678.exe_": { + "diffs": 0, + "details": [] + }, + "0cd2b334aede270b14868db28211cde3.exe_": { + "diffs": 2, + "details": [ + " rule missing in capa-x: 'contain loop'", + " rule missing in capa-x: 'packed with generic packer'" + ] + }, + "0db010298586f17ee7e46f390d5724be.exe_": { + "diffs": 0, + "details": [] + }, + "0f33c2fec223823f84a732ceb1ad94ed2645e896095144850cf1443aeda67da6.dll_": { + "diffs": 0, + "details": [] + }, + "1038a23daad86042c66bfe6c9d052d27048de9653bde5750dc0f240c792d9ac8.elf_": { + "diffs": 0, + "details": [] + }, + "10cd7afd580ee9c222b0a87ff241d306.dll_": { + "diffs": 0, + "details": [] + }, + "10ebcf8c20403457a08762200015b151.exe_": { + "diffs": 0, + "details": [] + }, + "112f9f0e8d349858a80dd8c14190e620.exe_": { + "diffs": 3, + "details": [ + " rule missing in capa-x: 'get keyboard layout'", + " rule missing in capa-x: 'log keystrokes via application hook'", + " rule missing in capa-x: 'set application hook'" + ] + }, + "138c71e94961fe9e31cec5dfba62a639.exe_": { + "diffs": 0, + "details": [] + }, + "15835b6dd703e69d22d4ab941ccd5f6e78c3abc22ae123366da5e950eaa62e2b.dll_": { + "diffs": 0, + "details": [] + }, + "1e2791877da02d49998dea79515a89ca.dll_": { + "diffs": 0, + "details": [] + }, + "1e9fc7f32bd5522dd0222932eb9f1d8bd0a2e132c7b46cfcc622ad97831e6128.exe_": { + "diffs": 1, + "details": [ + " rule extra in capa-x: 'delete registry key'" + ] + }, + "2055994ff75b4309eee3a49c5749d306.exe_": { + "diffs": 0, + "details": [] + }, + "22d0a2e4c9c2163b2bf5f0e41a2e1762.exe_": { + "diffs": 0, + "details": [] + }, + "253309d8b3675d3cc61d4bf23aa15d4b.dll_": { + "diffs": 1, + "details": [ + " rule missing in capa-x: 'enumerate gui resources'" + ] + }, + "26beba7352a32b803aa19e0782011a383a1df19549910e7b2f2f244e49678524.dll_": { + "diffs": 0, + "details": [] + }, + "276f691a3df25481f59d79781799e35f.exe_": { + "diffs": 3, + "details": [ + " rule missing in capa-x: 'compute adler32 checksum'", + " rule missing in capa-x: 'hash data with CRC32'", + " rule missing in capa-x: 'resolve function by parsing PE exports'" + ] + }, + "2826b762b9c268601a44974ef469a671b441e798a6c3cbb40070450c6c030ba2.exe_": { + "diffs": 2, + "details": [ + " rule extra in capa-x: '(internal) packer file limitation'", + " rule extra in capa-x: 'packed with Themida'" + ] + }, + "2855ba06b90e7c64d9bce888e47baf6d.exe_": { + "diffs": 0, + "details": [] + }, + "294b8db1f2702b60fb2e42fdc50c2cee6a5046112da9a5703a548a4fa50477bc.elf_": { + "diffs": 4, + "details": [ + " rule missing in capa-x: 'create process on Linux'", + " rule missing in capa-x: 'enumerate files on Linux'", + " rule missing in capa-x: 'set current directory'", + " rule missing in capa-x: 'write file on Linux'" + ] + }, + "29a76e413ffe0f78b62926b8455082d6.exe_": { + "diffs": 1, + "details": [ + " rule extra in capa-x: 'execute shellcode via indirect call'" + ] + }, + "2a7429d60040465f9bd27bbae2beef88.exe_": { + "diffs": 0, + "details": [] + }, + "2bf18d0403677378adad9001b1243211.elf_": { + "diffs": 0, + "details": [] + }, + "2d3edc218a90f03089cc01715a9f047f.exe_": { + "diffs": 0, + "details": [] + }, + "2ebadd04f0ada89c36c1409b6e96423a68dd77b513db8db3da203c36d3753e5f.exe_": { + "diffs": 0, + "details": [] + }, + "2f7f5fb5de175e770d7eae87666f9831.elf_": { + "diffs": 3, + "details": [ + " rule missing in capa-x: 'decrypt data using AES via x86 extensions'", + " rule missing in capa-x: 'generate random numbers using a Mersenne Twister'", + " rule missing in capa-x: 'parse credit card information'" + ] + }, + "31600ad0d1a7ea615690df111ae36c73.exe_": { + "diffs": 1, + "details": [ + " rule extra in capa-x: 'create directory'" + ] + }, + "31cee4f66cf3b537e3d2d37a71f339f4.exe_": { + "diffs": 0, + "details": [] + }, + "321338196a46b600ea330fc5d98d0699.exe_": { + "diffs": 0, + "details": [] + }, + "32b3678f8c29437e9ea10eab10194f66.exe_": { + "diffs": 0, + "details": [] + }, + "34404a3fb9804977c6ab86cb991fb130.exe_": { + "diffs": 0, + "details": [] + }, + "34dbc85ed0386e024c724c7969e8d0ff0ff0b1882508ea259c458d59657a1971.elf_": { + "diffs": 2, + "details": [ + " rule extra in capa-x: 'create UDP socket'", + " rule extra in capa-x: 'set file attributes'" + ] + }, + "3583f7f97ab207be7ab2ec0a507e2481.dll_": { + "diffs": 0, + "details": [] + }, + "35f9cfe5110471a82e330d904c97466a.dll_": { + "diffs": 1, + "details": [ + " rule extra in capa-x: 'delay execution'" + ] + }, + "368239d36d221d8877a07ab6799e643a.elf_": { + "diffs": 0, + "details": [] + }, + "36f506a34b99bf4c199b3c9ec8aa02bd631feafdca20e69e33e714c269ddb8c5.dll_": { + "diffs": 0, + "details": [] + }, + "3808f21e56dede99bc914d90aeabe47a.exe_": { + "diffs": 0, + "details": [] + }, + "39ce034911a6ebd482af5893f9bdbd95.exe_": { + "diffs": 0, + "details": [] + }, + "3aa7ee4d67f562933bc998f352b1f319.dll_": { + "diffs": 0, + "details": [] + }, + "3b13b6f1d7cd14dc4a097a12e2e505c0a4cff495262261e2bfc991df238b9b04.dll_": { + "diffs": 0, + "details": [] + }, + "3d0d79c4715e5b6dd2f8704e4878d57646d5a92e1116aec2ef81a5679ba45cd8.exe_": { + "diffs": 0, + "details": [] + }, + "3d5a99114b24e4373b77077072e4ed52ffce9e2607ec6fe4ed2f08d3b1c59026.dll_": { + "diffs": 0, + "details": [] + }, + "3d760b6fc84571c928bed835863fc302.exe_": { + "diffs": 0, + "details": [] + }, + "3da7c2c70a2d93ac4643f20339d5c7d61388bddd77a4a5fd732311efad78e535.elf_": { + "diffs": 0, + "details": [] + }, + "3fdfb2d522e7deecaaaf2f87420f7e75.exe_": { + "diffs": 1, + "details": [ + " rule extra in capa-x: 'link function at runtime on Windows'" + ] + }, + "44461306a604d2cd9883c2bb623af276.dll_": { + "diffs": 0, + "details": [] + }, + "44bad2e2a9e387b86870f009d01833ea4618d2a7cda5f64fa84a19f3bdf4efaf.exe_": { + "diffs": 0, + "details": [] + }, + "464ef2ca59782ce697bc329713698ccc.exe_": { + "diffs": 0, + "details": [] + }, + "48c7ad2d9d482cb11898f2719638ceed.exe_": { + "diffs": 0, + "details": [] + }, + "49a34cfbeed733c24392c9217ef46bb6.exe_": { + "diffs": 4, + "details": [ + " rule missing in capa-x: 'contain obfuscated stackstrings'", + " rule missing in capa-x: 'encrypt data using RC4 PRGA'", + " rule missing in capa-x: 'hash data with MD5'", + " rule missing in capa-x: 'parse credit card information'" + ] + }, + "4bdd67ff852c221112337fecd0681eac.exe_": { + "diffs": 0, + "details": [] + }, + "4e9c546a54e40d0da89bb4616dd7f8c4.exe_": { + "diffs": 0, + "details": [] + }, + "50d5ee1ce2ca5e30c6b1019ee64eeec2.exe_": { + "diffs": 2, + "details": [ + " rule extra in capa-x: 'delete directory'", + " rule extra in capa-x: 'enumerate files recursively'" + ] + }, + "512a5575ff395389fc952d7925e72be1ca4ba86a5d4f2da50f1cbbde0b208c92.elf_": { + "diffs": 3, + "details": [ + " rule missing in capa-x: 'calculate modulo 256 via x86 assembly'", + " rule missing in capa-x: 'encrypt data using RC4 PRGA'", + " rule missing in capa-x: 'use io_uring IO interface on Linux'" + ] + }, + "51828683dc26bfabd3994494099ae97d.elf_": { + "diffs": 0, + "details": [] + }, + "52d8e95c9883cd16d7b44e3a7adc22d6.exe_": { + "diffs": 0, + "details": [] + }, + "54ac78733552a62d1d05ea4ba3fc604bb49fe000d7fc948da45335b726e64d75.dll_": { + "diffs": 0, + "details": [] + }, + "5589857ba6ad8cd0893f4fbba774382222a573d20a2dd71b26b3b4d64f671301.exe_": { + "diffs": 0, + "details": [] + }, + "5789181cba467fa940cdce809b88b9bf7e0e9d4e079e32a68d2b911a8cc47da9.exe_": { + "diffs": 0, + "details": [] + }, + "58adc2e97fbee01b71073ccd7ff1b9a4.exe_": { + "diffs": 0, + "details": [] + }, + "592cfd22bba96ef3aab566fe7bf82aff5e1b4130856d1f7f847d03d4689af7e7.exe_": { + "diffs": 0, + "details": [] + }, + "5a2f620f29ca2f44fc22df67b674198f.exe_": { + "diffs": 0, + "details": [] + }, + "5b99fa01c72cebc53a76cc72e9581189.dll_": { + "diffs": 0, + "details": [] + }, + "5dd0b130d5c3d40c69e3972f39fd7d62.exe_": { + "diffs": 0, + "details": [] + }, + "5f66b82558ca92e54e77f216ef4c066c.exe_": { + "diffs": 0, + "details": [] + }, + "638dcc3d37b3a574044233c9637d7288.exe_": { + "diffs": 0, + "details": [] + }, + "648fc498110b11b4313a47a776e6ba40.exe_": { + "diffs": 0, + "details": [] + }, + "64e9f62840db2f65fc717cfaf99081f9.dll_": { + "diffs": 0, + "details": [] + }, + "659a539cfff56e5450af76f8696b7122d6a22b1d7dbe5121c7b6308eb6ec1c3a.exe_": { + "diffs": 0, + "details": [] + }, + "65b1ea6e5254d458c602504ceeda5e05.exe_": { + "diffs": 0, + "details": [] + }, + "67f8302a2fd28d15f62d6d20d748bfe350334e5353cbdef112bd1f8231b5599d.exe_": { + "diffs": 0, + "details": [] + }, + "6b25f1e754ef486bbb28a66d46bababe.exe_": { + "diffs": 0, + "details": [] + }, + "6c440a5ce8509984dcc4e703d0e4dd9bffc4efd769dc8543f8d2e0cd86452822.dll_": { + "diffs": 0, + "details": [] + }, + "6cc148363200798a12091b97a17181a1.exe_": { + "diffs": 0, + "details": [] + }, + "6ee9bb8b897c2d69f797646d5f94de0f.elf_": { + "diffs": 0, + "details": [] + }, + "6f99a2c8944cb02ff28c6f9ced59b161.exe_": { + "diffs": 0, + "details": [] + }, + "6f9cb3f56d227fd57f0b75220472d744a6de894e7f74302ae39bbb164a92cdd6.exe_": { + "diffs": 0, + "details": [] + }, + "70fd3347786ed7a4a43910e6778ef296.exe_": { + "diffs": 7, + "details": [ + " rule missing in capa-x: 'PEB access'", + " rule missing in capa-x: 'delete directory'", + " rule missing in capa-x: 'get OS version'", + " rule missing in capa-x: 'modify access privileges'", + " rule missing in capa-x: 'shutdown system'", + " rule extra in capa-x: 'allocate memory'", + " rule extra in capa-x: 'allocate or change RW memory'" + ] + }, + "71a4f9b800d81ff6632b9892a6a502c412c141341e46d697a8c004e2f460913b.exe_": { + "diffs": 0, + "details": [] + }, + "72c8e3c8049927fa1dd53b61d9b67b9d74ab5b6c030bc38dfbe9a338a32438c9.dll_": { + "diffs": 0, + "details": [] + }, + "7351f8a40c5450557b24622417fc478d.elf_": { + "diffs": 0, + "details": [] + }, + "749cf36adc5513c92c7acc836d20935e3c433f3c2d5641293e7a9c57c5ce22c2.elf_": { + "diffs": 4, + "details": [ + " rule missing in capa-x: 'calculate modulo 256 via x86 assembly'", + " rule extra in capa-x: 'create or open file'", + " rule extra in capa-x: 'map or unmap memory on Linux'", + " rule extra in capa-x: 'write file on Linux'" + ] + }, + "74e0758e469ab87f38cda7925d696ea41a122364891860e1ae30fc9ff2f68e7d.exe_": { + "diffs": 0, + "details": [] + }, + "74fa32d2b277f583010b692a3f91b627.exe_": { + "diffs": 0, + "details": [] + }, + "77d87e9937546aebc1595039d730352b15fab32c72a76913f04262c6802d098f.exe_": { + "diffs": 0, + "details": [] + }, + "79252f58d486aee8c08a8a7ebd36ae11ab5798b289e7f88e71eacf8637c340cc.dll_": { + "diffs": 0, + "details": [] + }, + "79cde1aa711e321b4939805d27e160be.exe_": { + "diffs": 0, + "details": [] + }, + "7d16efd0078f22c17a4bd78b0f0cc468.exe_": { + "diffs": 0, + "details": [] + }, + "7d333c9b11b06ef0982b61bfc062631bb6cf9d12d0d4f2cf1b807a25ddf62fbc.exe_": { + "diffs": 0, + "details": [] + }, + "7f15b1a47bbe031334e23653879e9661f4b8cde80c307548328fdd3aed87ca46.exe_": { + "diffs": 0, + "details": [] + }, + "7fbc17a09cf5320c515fc1c5ba42c8b3.exe_": { + "diffs": 0, + "details": [] + }, + "80372de850597bd9e7e021a94f13f0a1.exe_": { + "diffs": 0, + "details": [] + }, + "82bf6347acf15e5d883715dc289d8a2b.exe_": { + "diffs": 2, + "details": [ + " rule extra in capa-x: 'enumerate PE sections'", + " rule extra in capa-x: 'resolve function by parsing PE exports'" + ] + }, + "8333822ed41d9f2b302cf8e21b126efc.exe_": { + "diffs": 0, + "details": [] + }, + "84f1b049fa8962b215a77f51af6714b3.dll_": { + "diffs": 0, + "details": [] + }, + "86a8f267cf0f51c032f7b1777eb1e51f7cd1badf3f3894e2557a3f571fca9f3d.exe_": { + "diffs": 0, + "details": [] + }, + "873275ce8bf88ef66e9fa0c74b5c2a1e.exe_": { + "diffs": 0, + "details": [] + }, + "8a132663bee5c2f0f5cbfebee1b55ac72934632bf32bc32d6e2dae797c9e6e35.dll_": { + "diffs": 0, + "details": [] + }, + "8b1682c85612db9f62eb3600a90b54e83117bb756a7456c9f82ccdcb0bf4b7e4.exe_": { + "diffs": 0, + "details": [] + }, + "91a12a4cf437589ba70b1687f5acad19.exe_": { + "diffs": 0, + "details": [] + }, + "9324d1a8ae37a36ae560c37448c9705a.exe_": { + "diffs": 2, + "details": [ + " rule extra in capa-x: 'connect to URL'", + " rule extra in capa-x: 'create HTTP request'" + ] + }, + "932dab8756ad4ae9a62bde0772d952e4.exe_": { + "diffs": 0, + "details": [] + }, + "9486f2c5d514c1f39833b852ab03f4c0297e9e82b456b0dccad1a2d7d15c3385.elf_": { + "diffs": 0, + "details": [] + }, + "971e599e6e707349eccea2fd4c8e5f67.exe_": { + "diffs": 2, + "details": [ + " rule missing in capa-x: 'encode data using XOR'", + " rule extra in capa-x: 'PEB access'" + ] + }, + "98c37c3c23bbfb362dac7754c6ba48e75cf24d73bc963a4cdfca557b9e016909.exe_": { + "diffs": 2, + "details": [ + " rule missing in capa-x: 'get file size'", + " rule missing in capa-x: 'write file on Windows'" + ] + }, + "9a00ebe67d833edb70ed6dd0f4652592.dll_": { + "diffs": 0, + "details": [] + }, + "9b7ccaa2ae6a5b96e3110ebcbc4311f6.dll_": { + "diffs": 3, + "details": [ + " rule missing in capa-x: 'create or open registry key'", + " rule missing in capa-x: 'encrypt or decrypt via WinCrypt'", + " rule missing in capa-x: 'get service handle'" + ] + }, + "9bca6b99e7981208af4c7925b96fb9cf.exe_": { + "diffs": 0, + "details": [] + }, + "9d98f8519d9fee8219caca5b31eef0bd.exe_": { + "diffs": 0, + "details": [] + }, + "9efa86b43b4367bcdc1591aee59bda25.dll_": { + "diffs": 0, + "details": [] + }, + "9ff8e68343cc29c1036650fc153e69f7.exe_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 01-01.exe_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 01-02.exe_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 03-02.dll_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 03-03.exe_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 05-01.dll_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 10-03.sys_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 11-03.dll_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 12-01.dll_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 12-02.exe_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 12-03.exe_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 14-01.exe_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 14-02.exe_": { + "diffs": 3, + "details": [ + " rule missing in capa-x: 'read file on Windows'", + " rule missing in capa-x: 'read pipe'", + " rule missing in capa-x: 'write file on Windows'" + ] + }, + "Practical Malware Analysis Lab 16-02.exe_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 17-02.dll_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 17-03.exe_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 18-03.exe_": { + "diffs": 0, + "details": [] + }, + "Practical Malware Analysis Lab 20-02.exe_": { + "diffs": 0, + "details": [] + }, + "a0fb20bc9aa944c3a0a6c4545c195818.exe_": { + "diffs": 0, + "details": [] + }, + "a198216798ca38f280dc413f8c57f2c2.exe_": { + "diffs": 0, + "details": [] + }, + "a210a5daaf487fe6c8bbaf906abce749042f15890d60b09c6cb333e54958663b.dll_": { + "diffs": 0, + "details": [] + }, + "a30101595f6f28ab2f4b0b2cd177c3c4d2ab34a355ab7761a3795d0887c24ada.exe_": { + "diffs": 3, + "details": [ + " rule missing in capa-x: 'allocate thread local storage'", + " rule missing in capa-x: 'encode data using XOR'", + " rule missing in capa-x: 'hash data using djb2'" + ] + }, + "a45e377dbb98a6b44fd4034bc3fff9b0.exe_": { + "diffs": 0, + "details": [] + }, + "a563c50c5fa0fd541248acaf72cc4e7d.exe_": { + "diffs": 3, + "details": [ + " rule missing in capa-x: 'allocate thread local storage'", + " rule extra in capa-x: 'enumerate files on Windows'", + " rule extra in capa-x: 'enumerate files recursively'" + ] + }, + "a6594d9550d56ddeaac8b3140821e698eefb7163ba29f0119c2ef19beb6040b0.exe_": { + "diffs": 1, + "details": [ + " rule extra in capa-x: 'encrypt data using Salsa20 or ChaCha'" + ] + }, + "a6e9d94e286984b9e44c58df5eeb20cf5c08b9acff4af341ddecdfbe3ea9249d.elf_": { + "diffs": 1, + "details": [ + " rule extra in capa-x: 'get current process memory mapping on Linux'" + ] + }, + "a70052c45e907820187c7e6bcdc7ecca.exe_": { + "diffs": 7, + "details": [ + " rule missing in capa-x: 'calculate modulo 256 via x86 assembly'", + " rule missing in capa-x: 'encode data using XOR'", + " rule missing in capa-x: 'encrypt data using RC4 PRGA'", + " rule missing in capa-x: 'enumerate PE sections'", + " rule missing in capa-x: 'receive data on socket'", + " rule missing in capa-x: 'resolve function by parsing PE exports'", + " rule missing in capa-x: 'send data on socket'" + ] + }, + "a72ac9f1cf6cc01103765f866aae2dd85ea48208fb180c01076ca982684a4032.elf_": { + "diffs": 0, + "details": [] + }, + "a90e5b3454aa71d9700b2ea54615f44b.exe_": { + "diffs": 0, + "details": [] + }, + "ad4229879180e267f431ac6666b6a0a2.exe_": { + "diffs": 1, + "details": [ + " rule missing in capa-x: 'compute adler32 checksum'" + ] + }, + "af13e7583ed1b27c4ae219e344a37e2b.exe_": { + "diffs": 0, + "details": [] + }, + "al-khaser_x86.exe_": { + "diffs": 0, + "details": [] + }, + "b1133d7e5599beefe992a127f6e9704176a13b7e86b4db45c3b61cf25a60d414.exe_": { + "diffs": 0, + "details": [] + }, + "b2ad4409323147b63e370745e5209996.exe_": { + "diffs": 0, + "details": [] + }, + "b5f0524e69b3a3cf636c7ac366ca57bf5e3a8fdc8a9f01caf196c611a7918a87.elf_": { + "diffs": 3, + "details": [ + " rule missing in capa-x: 'get file attributes'", + " rule missing in capa-x: 'move file'", + " rule missing in capa-x: 'set socket configuration'" + ] + }, + "b761e060c7114448d6a5fe276d4ca882c4bd702c12c4d73f6ad79b8dfac33448.exe_": { + "diffs": 0, + "details": [] + }, + "b7b5e1253710d8927cbe07d52d2d2e10.exe_": { + "diffs": 0, + "details": [] + }, + "b83480162ede09d4aa6d4850f9faa0a4c3834152752fd04cfdb22d647aa1f825.exe_": { + "diffs": 0, + "details": [] + }, + "ba947eb07d8c823949316a97364d060f.exe_": { + "diffs": 0, + "details": [] + }, + "bb38149ff4b5c95722b83f24ca27a42b.elf_": { + "diffs": 0, + "details": [] + }, + "bc452cc1128ccf7fa9f76d83cda79132740414973600fed14509749fe946816e.exe_": { + "diffs": 0, + "details": [] + }, + "bf7a9c8bdfa6d47e01ad2b056264acc3fd90cf43fe0ed8deec93ab46b47d76cb.elf_": { + "diffs": 0, + "details": [] + }, + "bf88e1bd4a3bde10b419a622278f1ff7.exe_": { + "diffs": 1, + "details": [ + " rule missing in capa-x: 'allocate thread local storage'" + ] + }, + "bfb9b5391a13d0afd787e87ab90f14f5.dll_": { + "diffs": 0, + "details": [] + }, + "c1969efd1e2be79909b880f4dbb8725e52efca82236f8a2165c5a8245393fcd6.exe_": { + "diffs": 0, + "details": [] + }, + "c2ba065654f13612ae63bca7f972ea91c6fe97291caeaaa3a28a180fb1912b3a.dll_": { + "diffs": 0, + "details": [] + }, + "c2bb17c12975ea61ff43a71afd9c3ff111d018af161859abae0bdb0b3dae98f9.exe_": { + "diffs": 0, + "details": [] + }, + "c2c3b3eea177b9411bc92a8800b40529fcd2d5c3696e71cbb2f4025429b314ee.elf_": { + "diffs": 0, + "details": [] + }, + "c2d46d256b8f9490c9599eea11ecef19fde7d4fdd2dea93604cee3cea8e172ac.exe_": { + "diffs": 1, + "details": [ + " rule extra in capa-x: 'contain loop'" + ] + }, + "c335a9d41185a32ad918c5389ee54235.exe_": { + "diffs": 1, + "details": [ + " rule missing in capa-x: 'delete registry key'" + ] + }, + "c3699d0c7bd1276184249a487c1f0d7f.exe_": { + "diffs": 4, + "details": [ + " rule missing in capa-x: 'get session user name'", + " rule missing in capa-x: 'hide graphical window'", + " rule missing in capa-x: 'read pipe'", + " rule missing in capa-x: 'receive data on socket'" + ] + }, + "c66172b12971a329f8d5ff01665f204b.exe_": { + "diffs": 0, + "details": [] + }, + "cb948b13a5046a692ec3ed8cc16a9566.exe_": { + "diffs": 0, + "details": [] + }, + "ccbf7cba35bab56563c0fbe4237fdc41.exe_": { + "diffs": 0, + "details": [] + }, + "cd2cba9e6313e8df2c1273593e649682.exe_": { + "diffs": 0, + "details": [] + }, + "d063b1804e8d2bb26bd2e097141c1bbc.exe_": { + "diffs": 0, + "details": [] + }, + "d7ff81ff775d4ab50d31ac1e962c8c4dea7ff9f280aa2b42ddd06760a5665002.exe_": { + "diffs": 0, + "details": [] + }, + "d8d2ed4a5f13e4bda8edf89d0dd1ef57cec8b18c01fcccdb89fb745a1a2be05f.exe_": { + "diffs": 0, + "details": [] + }, + "d9531e53036c5d04fbe7d1aeae2988c3bf0fdec63774690c5df70cc121af8de4.dll_": { + "diffs": 0, + "details": [] + }, + "d9630c174b8ff5c0aa26168df523e63e.exe_": { + "diffs": 0, + "details": [] + }, + "db9fe790b4e18abf55df31aa0b81e558.exe_": { + "diffs": 0, + "details": [] + }, + "dc7cb53c5dc2e756822328a7144c29318cb871890727eff9c8da64a01e8e782d.dll_": { + "diffs": 0, + "details": [] + }, + "df814d4b55912e4ba404c62080b3a7eda70a3c6283ea740f8a14a9116d803259.dll_": { + "diffs": 0, + "details": [] + }, + "e17e6a79ed614f5468d0eed758629697.elf_": { + "diffs": 2, + "details": [ + " rule extra in capa-x: 'authenticate HMAC'", + " rule extra in capa-x: 'parse credit card information'" + ] + }, + "e353d3fbfb5c3738a77a622adff9a416.exe_": { + "diffs": 0, + "details": [] + }, + "e4c33ac3638eef68311f8ac0d72483c7.exe_": { + "diffs": 0, + "details": [] + }, + "e59ffeaf7acb0c326e452fa30bb71a36.exe_": { + "diffs": 0, + "details": [] + }, + "e5e8c139772efe47f738f4788ae9b3dc97960b1c006bc6a406715cab69f27cfc.elf_": { + "diffs": 1, + "details": [ + " rule extra in capa-x: 'contain loop'" + ] + }, + "e87076a1182ba40758e1d7258442c1ee23bf71ac77fad9f3babc707dce11c144.exe_": { + "diffs": 0, + "details": [] + }, + "ea2876e9175410b6f6719f80ee44b9553960758c7d0f7bed73c0fe9a78d8e669.dll_": { + "diffs": 0, + "details": [] + }, + "ea7bb99e03606702c1cbe543bb32b27e.dll_": { + "diffs": 0, + "details": [] + }, + "eb355bd63bddce02955792b4cd6539fb.dll_": { + "diffs": 0, + "details": [] + }, + "efd1a86330cecba5d8d038fba65ac8e76955ed724986aa87cd6ca9f72f6941c7.dll_": { + "diffs": 0, + "details": [] + }, + "f4584aed97469c461826ea17ca9b3c1d.exe_": { + "diffs": 0, + "details": [] + }, + "f53dfa294d6979145fbb34303cf73bb6.exe_": { + "diffs": 1, + "details": [ + " rule extra in capa-x: 'decode data using Base64 via VBMI lookup table'" + ] + }, + "f59035192098e44b86c4648a0de4078edbe80352260276f4755d15d354f5fc58.exe_": { + "diffs": 2, + "details": [ + " rule missing in capa-x: 'hash data using SHA256'", + " rule missing in capa-x: 'log keystrokes via Input Method Manager'" + ] + }, + "f9ac6b16273556b3a57bf2c6d7e7db97.exe_": { + "diffs": 0, + "details": [] + }, + "ffa48ed4b7b48897f6756c4222b2606399de0bca627cedfddf61e69986580430.elf_": { + "diffs": 0, + "details": [] + }, + "ffeae4a391a1d5203bd04b4161557227.exe_": { + "diffs": 0, + "details": [] + }, + "microsocks.elf_": { + "diffs": 0, + "details": [] + } + } +} diff --git a/scripts/difftest.py b/scripts/difftest.py index f466c41..503e245 100755 --- a/scripts/difftest.py +++ b/scripts/difftest.py @@ -1421,7 +1421,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--capa-cli", type=Path, - default=REPO_ROOT / "target" / "debug" / "capa", + default=REPO_ROOT / "target" / "debug" / "capa-x", help="path to the capa-x binary (default: debug build)", ) args = parser.parse_args(argv)