feat(encode): faster entropy stage and a lazy match-finder switch

.github/workflows/ci.yml

@@ -211,6 +211,29 @@ jobs:
             echo "::error::bench-only instrumentation symbols leaked into zstd/src/"
             exit 1
           fi
+      - name: Gate — one search per tier in the debug build
+        # Below the encoder's one tier choice, code is compiled per tier. A
+        # plain `match` on `K::TIER` instantiates every arm's kernel for every
+        # tier in an unoptimised build (`tier_match!` does not), and a test
+        # closure handed straight to a search compiles that search again per
+        # closure (`dyn_sink` does not). Either one multiplies the test build's
+        # IR until the compiler no longer fits on a hosted runner.
+        # `fastpath/mod.rs` defines `tier_match!` itself, whose exhaustiveness
+        # check is the one `match` on the constant that belongs there.
+        run: |
+          found=$(rg -n --no-heading -g '!zstd/src/encoding/fastpath/mod.rs' \
+              -e 'match\s+[A-Za-z_]+::TIER\b' \
+              -e 'match\s+<[^>]+>::TIER\b' \
+              -e '::TIER\s*[=!]=' \
+              zstd/src/ || true)
+          found+=$(rg -n --no-heading -g '*tests*.rs' \
+              -e 'start_matching\w*::<K>\(\s*\|' \
+              zstd/src/ || true)
+          if [ -n "$found" ]; then
+            echo "$found"
+            echo "::error::per-tier code must pick its kernel with tier_match! and take test sinks through dyn_sink"
+            exit 1
+          fi
 
   c-abi:
     # Builds the libzstd-compatible C ABI front end and verifies it is a real

README.md

@@ -389,13 +389,13 @@ benefits.
 |-------------|----------|---------|
 | 1-2         | `Fast`     | `Simple` matcher |
 | 3-4         | `Dfast`    | `Dfast` two-tier hash |
-| 5-12        | `Greedy` / `Lazy` / `Lazy2` | `Row` lazy parse (`lazy_depth=0/1/2`): row match-finder above a 2^14 window, hash chain at or below it |
+| 5-12        | `Greedy` / `Lazy` / `Lazy2` | `Row` lazy parse (`lazy_depth=0/1/2`): row match-finder above a 2^14 window, hash chain at or below it, unless `RowMatchFinderMode` forces one |
 | 13-15       | `Btlazy2`  | `Row` lazy parse over the lazily-sorted binary tree |
-| 16-17       | `BtOpt`    | `HashChain` candidates + `btopt` price parser |
-| 18          | `BtUltra`  | `HashChain` candidates + `btultra` price parser |
-| 19-22       | `BtUltra2` | `HashChain` candidates + `btultra2` dual-profile parse |
+| 16-17       | `BtOpt`    | binary-tree candidates + `btopt` price parser |
+| 18          | `BtUltra`  | binary-tree candidates + `btultra` price parser |
+| 19-22       | `BtUltra2` | binary-tree candidates + `btultra2` dual-profile parse |
 
-The level → strategy column matches upstream zstd `ZSTD_defaultCParameters[0]` at `zstd/lib/compress/clevels.h:25-50` (srcSize > 256 KiB tier); smaller sources shift the row per upstream's size tiers. The whole greedy..btlazy2 band runs upstream's `ZSTD_compressBlock_lazy_generic` parse on the `Row` backend over the three upstream match finders (rows / hash chain per `ZSTD_resolveRowMatchFinderMode`, lazily-sorted binary tree for `btlazy2`).
+The level → strategy column matches upstream zstd `ZSTD_defaultCParameters[0]` at `zstd/lib/compress/clevels.h:25-50` (srcSize > 256 KiB tier); smaller sources shift the row per upstream's size tiers. The whole greedy..btlazy2 band runs upstream's `ZSTD_compressBlock_lazy_generic` parse on the `Row` backend over the three upstream match finders (rows / hash chain per `ZSTD_resolveRowMatchFinderMode`, which `CompressionParametersBuilder::row_match_finder` and `ZSTD_c_useRowMatchFinder` override; lazily-sorted binary tree for `btlazy2`).
 
 </details>
 

c-api/src/cctx_params.rs

@@ -4,7 +4,9 @@
 
 use core::ffi::c_int;
 
-use codec::encoding::{CompressionLevel, CompressionParameters, LiteralCompressionMode, Strategy};
+use codec::encoding::{
+    CompressionLevel, CompressionParameters, LiteralCompressionMode, RowMatchFinderMode, Strategy,
+};
 
 use crate::cdict::ZSTD_compressionParameters;
 use crate::context::{ZSTD_CCtx, free_boxed, try_box};
@@ -46,7 +48,7 @@ impl ParamSwitch {
 /// searches, schedules or checks its work, with no counterpart in this
 /// encoder: stored as set so they read back, without effect on the frames.
 /// The sequence-format ones are read by the sequence entry points.
-const ADVISORY: [c_int; 18] = [
+const ADVISORY: [c_int; 17] = [
     ZSTD_C_NB_WORKERS,
     ZSTD_C_JOB_SIZE,
     ZSTD_C_OVERLAP_LOG,
@@ -59,7 +61,6 @@ const ADVISORY: [c_int; 18] = [
     ZSTD_C_BLOCK_DELIMITERS,
     ZSTD_C_VALIDATE_SEQUENCES,
     ZSTD_C_SPLIT_AFTER_SEQUENCES,
-    ZSTD_C_USE_ROW_MATCH_FINDER,
     ZSTD_C_DETERMINISTIC_REF_PREFIX,
     ZSTD_C_PREFETCH_CDICT_TABLES,
     ZSTD_C_ENABLE_SEQ_PRODUCER_FALLBACK,
@@ -105,6 +106,8 @@ pub(crate) struct CCtxParams {
     pub(crate) magicless: bool,
     /// `ZSTD_c_literalCompressionMode`.
     pub(crate) literal_compression: LiteralCompressionMode,
+    /// `ZSTD_c_useRowMatchFinder`.
+    pub(crate) row_match_finder: RowMatchFinderMode,
     /// `ZSTD_c_srcSizeHint`; 0 = none.
     pub(crate) src_size_hint: c_int,
     /// The [`ADVISORY`] parameters, in that order.
@@ -147,6 +150,7 @@ impl CCtxParams {
             max_block_size: 0,
             magicless: false,
             literal_compression: LiteralCompressionMode::Auto,
+            row_match_finder: RowMatchFinderMode::Auto,
             src_size_hint: 0,
             advisory: [0; ADVISORY.len()],
             pledged_src_size: CONTENTSIZE_UNKNOWN,
@@ -179,7 +183,8 @@ impl CCtxParams {
     /// (should be unreachable: every knob was bounds-checked on entry).
     pub(crate) fn resolve(&self) -> Option<CompressionParameters> {
         let mut builder = CompressionParameters::builder(CompressionLevel::from_level(self.level))
-            .literal_compression(self.literal_compression);
+            .literal_compression(self.literal_compression)
+            .row_match_finder(self.row_match_finder);
         if let Some(v) = self.window_log {
             builder = builder.window_log(v);
         }
@@ -301,6 +306,13 @@ impl CCtxParams {
                     _ => LiteralCompressionMode::Auto,
                 };
             }
+            ZSTD_C_USE_ROW_MATCH_FINDER => {
+                self.row_match_finder = match ParamSwitch::from_c(within(value)?) {
+                    Some(ParamSwitch::Enable) => RowMatchFinderMode::Enable,
+                    Some(ParamSwitch::Disable) => RowMatchFinderMode::Disable,
+                    _ => RowMatchFinderMode::Auto,
+                };
+            }
             // Clamped into range as upstream's multi-threaded build stores them
             // (zstd_compress.c:866-908); frames are still built on the calling
             // thread, which a worker count only permits, never requires.
@@ -371,6 +383,11 @@ impl CCtxParams {
                 LiteralCompressionMode::Enable => ParamSwitch::Enable.to_c(),
                 LiteralCompressionMode::Disable => ParamSwitch::Disable.to_c(),
             },
+            ZSTD_C_USE_ROW_MATCH_FINDER => match self.row_match_finder {
+                RowMatchFinderMode::Auto => ParamSwitch::Auto.to_c(),
+                RowMatchFinderMode::Enable => ParamSwitch::Enable.to_c(),
+                RowMatchFinderMode::Disable => ParamSwitch::Disable.to_c(),
+            },
             _ => match advisory_slot(param) {
                 Some(slot) => self.advisory[slot],
                 None => return Err(ZSTD_ErrorCode::ZSTD_error_parameter_unsupported),

c-api/src/tests/cctx_params.rs

@@ -45,6 +45,7 @@ const FORCE_MAX_WINDOW: c_int = 1000;
 const FORCE_ATTACH_DICT: c_int = 1001;
 const LITERAL_COMPRESSION_MODE: c_int = 1002;
 const SRC_SIZE_HINT: c_int = 1004;
+const USE_ROW_MATCH_FINDER: c_int = 1011;
 const MAX_BLOCK_SIZE: c_int = 1015;
 const BLOCK_SPLITTER_LEVEL: c_int = 1017;
 
@@ -421,6 +422,60 @@ fn literal_compression_mode_reaches_the_encoder() {
     assert_eq!(decompress(&raw, text.len()), text);
 }
 
+/// `ZSTD_c_useRowMatchFinder` picks the greedy / lazy band's finder: `enable`
+/// writes what `auto` writes on a source wide enough for rows and `disable`
+/// what `auto` writes on one of 16 KiB or less (upstream
+/// `ZSTD_resolveRowMatchFinderMode`), the forced choice differs from the
+/// window's, and every frame decodes. A value past the switch is refused.
+#[test]
+fn use_row_match_finder_reaches_the_encoder() {
+    let text = |lines: u32| -> Vec<u8> {
+        let mut state = 0x9E37_79B9u32;
+        let mut out = Vec::new();
+        for i in 0..lines {
+            state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
+            out.extend_from_slice(
+                format!(
+                    "{i:06} level=info shard={} took={}ms\n",
+                    state % 13,
+                    state >> 22
+                )
+                .as_bytes(),
+            );
+        }
+        out
+    };
+    let frame = |input: &[u8], mode: c_int| {
+        let cctx = ZSTD_createCCtx();
+        unsafe {
+            assert_eq!(ZSTD_CCtx_setParameter(cctx, LEVEL, 7), 0);
+            assert_eq!(ZSTD_CCtx_setParameter(cctx, USE_ROW_MATCH_FINDER, mode), 0);
+            assert_eq!(get_cctx(cctx, USE_ROW_MATCH_FINDER), mode);
+        }
+        let out = compress2(cctx, input);
+        unsafe { ZSTD_freeCCtx(cctx) };
+        assert_eq!(decompress(&out, input.len()), input, "mode {mode}");
+        out
+    };
+    let (auto, enable, disable) = (0, 1, 2);
+    let wide = text(6000);
+    assert_eq!(frame(&wide, enable), frame(&wide, auto));
+    assert_ne!(frame(&wide, disable), frame(&wide, auto));
+    let narrow = text(200);
+    assert!(narrow.len() <= 1 << 14);
+    assert_eq!(frame(&narrow, disable), frame(&narrow, auto));
+    assert_ne!(frame(&narrow, enable), frame(&narrow, auto));
+
+    let cctx = ZSTD_createCCtx();
+    unsafe {
+        assert_eq!(
+            code(ZSTD_CCtx_setParameter(cctx, USE_ROW_MATCH_FINDER, 3)),
+            ZSTD_ErrorCode::ZSTD_error_parameter_outOfBound
+        );
+        ZSTD_freeCCtx(cctx);
+    }
+}
+
 /// A size hint stays for every streamed frame and can be cleared again.
 #[test]
 fn src_size_hint_round_trips_streamed_frames() {

zstd/src/bin/structured-zstd/main.rs

@@ -17,7 +17,7 @@ use std::path::{Path, PathBuf};
 
 use structured_zstd::encoding::{
     CompressionContext, CompressionLevel, CompressionParameters, LevelParameters,
-    LiteralCompressionMode, Strategy, StreamingEncoder,
+    LiteralCompressionMode, RowMatchFinderMode, Strategy, StreamingEncoder,
 };
 
 /// Error type for the tool: a boxed message, which is all a command-line
@@ -341,6 +341,8 @@ struct Options {
     advanced: AdvancedParams,
     /// Whether literals are entropy-coded (`--[no-]compress-literals`).
     literals: LiteralCompressionMode,
+    /// The greedy / lazy band's finder (`--[no-]row-match-finder`).
+    row_match_finder: RowMatchFinderMode,
     /// Reference file for `--patch-from`: raw-content dictionary compression
     /// with the window sized to the input, so the whole reference is
     /// reachable.
@@ -1498,6 +1500,7 @@ fn parse_args_into(
         progress: Progress::Auto,
         advanced: AdvancedParams::default(),
         literals: LiteralCompressionMode::Auto,
+        row_match_finder: RowMatchFinderMode::Auto,
         patch_from: None,
         show_default_cparams: false,
         trainer: Trainer::FastCover,
@@ -1612,22 +1615,16 @@ fn parse_args_into(
                     return Ok(Parsed::Handled { pause: *pause });
                 }
                 // Flags that steer HOW the work is done, not what comes out:
-                // thread counts, IO strategy, matcher hints. We are
+                // thread counts, IO strategy. We are
                 // single-threaded and pick our own limits, so accepting them
                 // yields the same valid stream. Upstream takes them, so a
                 // script that passes them must not fail here: that is the
                 // whole drop-in contract.
                 "single-thread" => opts.threads = Some(0),
                 "adapt" => opts.adapt = true,
                 "rsyncable" => opts.rsyncable = true,
-                "sparse"
-                | "no-sparse"
-                | "asyncio"
-                | "no-asyncio"
-                | "mmap-dict"
-                | "no-mmap-dict"
-                | "row-match-finder"
-                | "no-row-match-finder" => {}
+                "sparse" | "no-sparse" | "asyncio" | "no-asyncio" | "mmap-dict"
+                | "no-mmap-dict" => {}
                 // The reference prints a trace of its own file-system helpers
                 // by their C names (`UTIL_stat`, `UTIL_isLink`, ...). There is
                 // nothing of ours that output would describe, and taking the
@@ -1643,6 +1640,8 @@ fn parse_args_into(
                 "fake-stderr-is-console" => opts.console.stderr = true,
                 "compress-literals" => opts.literals = LiteralCompressionMode::Enable,
                 "no-compress-literals" => opts.literals = LiteralCompressionMode::Disable,
+                "row-match-finder" => opts.row_match_finder = RowMatchFinderMode::Enable,
+                "no-row-match-finder" => opts.row_match_finder = RowMatchFinderMode::Disable,
                 _ => {
                     if long == "fast" {
                         // `--fast` is the level -1 alias.
@@ -2310,6 +2309,8 @@ Advanced compression options:
   --no-dictID                   Don't write `dictID` into the header (dictionary compression only).
   --[no-]content-size           Write the input size into the frame header when it is known. [Default: Write]
   --[no-]compress-literals      Force (un)compressed literals.
+  --[no-]row-match-finder       Explicitly enable/disable the fast, row-based matchfinder for
+                                the 'greedy', 'lazy', and 'lazy2' strategies.
 
   --format=zstd                 Compress files to the `.zst` format. [Default]
 
@@ -2347,8 +2348,7 @@ Environment: ZSTD_CLEVEL sets the default compression level; ZSTD_NBTHREADS is r
 Accepted for compatibility, with no effect here: -T#/--threads=#, --single-thread,
 --auto-threads, -B# and --block-size=# when compressing (the job size of a
 multi-threaded run), --adapt, --rsyncable, --zstd=ovlog=#, --[no-]sparse,
---[no-]asyncio, --[no-]mmap-dict, --[no-]row-match-finder (compression runs
-single-threaded). --adapt and --rsyncable are refused beside --single-thread or
+--[no-]asyncio, --[no-]mmap-dict (compression runs single-threaded). --adapt and --rsyncable are refused beside --single-thread or
 -T0 when compressing.
 
 Rejected rather than ignored, because they would change the result: --format=
@@ -6291,6 +6291,8 @@ struct FrameSettings {
     advanced: AdvancedParams,
     /// Whether literals are entropy-coded (`--[no-]compress-literals`).
     literals: LiteralCompressionMode,
+    /// The greedy / lazy band's finder (`--[no-]row-match-finder`).
+    row_match_finder: RowMatchFinderMode,
 }
 
 impl Default for FrameSettings {
@@ -6310,6 +6312,7 @@ impl Default for FrameSettings {
             dict_id_flag: true,
             advanced: AdvancedParams::default(),
             literals: LiteralCompressionMode::Auto,
+            row_match_finder: RowMatchFinderMode::Auto,
         }
     }
 }
@@ -6330,21 +6333,26 @@ impl FrameSettings {
             dict_id_flag: opts.dict_id_flag,
             advanced: opts.advanced,
             literals: opts.literals,
+            row_match_finder: opts.row_match_finder,
         }
     }
 }
 
 /// The fine-grained parameters a frame runs under, or `None` when nothing
 /// overrides the level: `--zstd=` knobs, `--long` (whose window yields to a
-/// `--zstd=wlog=`, as the reference command has it), and the literal mode.
-/// The long-distance knobs from `--zstd=` reach the encoder only with
-/// `--long`, since there they tune a matcher that otherwise does not run.
+/// `--zstd=wlog=`, as the reference command has it), and the literal and
+/// match-finder modes. The long-distance knobs from `--zstd=` reach the
+/// encoder only with `--long`, since there they tune a matcher that otherwise
+/// does not run.
 fn frame_parameters(
     level: CompressionLevel,
     settings: &FrameSettings,
 ) -> Result<Option<CompressionParameters>> {
     let advanced = settings.advanced;
-    if !settings.long && advanced.is_default() && settings.literals == LiteralCompressionMode::Auto
+    if !settings.long
+        && advanced.is_default()
+        && settings.literals == LiteralCompressionMode::Auto
+        && settings.row_match_finder == RowMatchFinderMode::Auto
     {
         return Ok(None);
     }
@@ -6388,7 +6396,9 @@ fn frame_parameters(
             builder = builder.ldm_hash_rate_log(log);
         }
     }
-    builder = builder.literal_compression(settings.literals);
+    builder = builder
+        .literal_compression(settings.literals)
+        .row_match_finder(settings.row_match_finder);
     builder
         .build()
         .map(Some)