diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25c7b5e8d..0716b83d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -211,6 +211,29 @@ jobs: echo "::error::bench-only instrumentation symbols leaked into zstd/src/" exit 1 fi + - name: Gate — one search per tier in the debug build + # Below the encoder's one tier choice, code is compiled per tier. A + # plain `match` on `K::TIER` instantiates every arm's kernel for every + # tier in an unoptimised build (`tier_match!` does not), and a test + # closure handed straight to a search compiles that search again per + # closure (`dyn_sink` does not). Either one multiplies the test build's + # IR until the compiler no longer fits on a hosted runner. + # `fastpath/mod.rs` defines `tier_match!` itself, whose exhaustiveness + # check is the one `match` on the constant that belongs there. + run: | + found=$(rg -n --no-heading -g '!zstd/src/encoding/fastpath/mod.rs' \ + -e 'match\s+[A-Za-z_]+::TIER\b' \ + -e 'match\s+<[^>]+>::TIER\b' \ + -e '::TIER\s*[=!]=' \ + zstd/src/ || true) + found+=$(rg -n --no-heading -g '*tests*.rs' \ + -e 'start_matching\w*::\(\s*\|' \ + zstd/src/ || true) + if [ -n "$found" ]; then + echo "$found" + echo "::error::per-tier code must pick its kernel with tier_match! and take test sinks through dyn_sink" + exit 1 + fi c-abi: # Builds the libzstd-compatible C ABI front end and verifies it is a real diff --git a/README.md b/README.md index 51e2fbebd..f9ae34190 100644 --- a/README.md +++ b/README.md @@ -389,13 +389,13 @@ benefits. |-------------|----------|---------| | 1-2 | `Fast` | `Simple` matcher | | 3-4 | `Dfast` | `Dfast` two-tier hash | -| 5-12 | `Greedy` / `Lazy` / `Lazy2` | `Row` lazy parse (`lazy_depth=0/1/2`): row match-finder above a 2^14 window, hash chain at or below it | +| 5-12 | `Greedy` / `Lazy` / `Lazy2` | `Row` lazy parse (`lazy_depth=0/1/2`): row match-finder above a 2^14 window, hash chain at or below it, unless `RowMatchFinderMode` forces one | | 13-15 | `Btlazy2` | `Row` lazy parse over the lazily-sorted binary tree | -| 16-17 | `BtOpt` | `HashChain` candidates + `btopt` price parser | -| 18 | `BtUltra` | `HashChain` candidates + `btultra` price parser | -| 19-22 | `BtUltra2` | `HashChain` candidates + `btultra2` dual-profile parse | +| 16-17 | `BtOpt` | binary-tree candidates + `btopt` price parser | +| 18 | `BtUltra` | binary-tree candidates + `btultra` price parser | +| 19-22 | `BtUltra2` | binary-tree candidates + `btultra2` dual-profile parse | -The level → strategy column matches upstream zstd `ZSTD_defaultCParameters[0]` at `zstd/lib/compress/clevels.h:25-50` (srcSize > 256 KiB tier); smaller sources shift the row per upstream's size tiers. The whole greedy..btlazy2 band runs upstream's `ZSTD_compressBlock_lazy_generic` parse on the `Row` backend over the three upstream match finders (rows / hash chain per `ZSTD_resolveRowMatchFinderMode`, lazily-sorted binary tree for `btlazy2`). +The level → strategy column matches upstream zstd `ZSTD_defaultCParameters[0]` at `zstd/lib/compress/clevels.h:25-50` (srcSize > 256 KiB tier); smaller sources shift the row per upstream's size tiers. The whole greedy..btlazy2 band runs upstream's `ZSTD_compressBlock_lazy_generic` parse on the `Row` backend over the three upstream match finders (rows / hash chain per `ZSTD_resolveRowMatchFinderMode`, which `CompressionParametersBuilder::row_match_finder` and `ZSTD_c_useRowMatchFinder` override; lazily-sorted binary tree for `btlazy2`). diff --git a/c-api/src/cctx_params.rs b/c-api/src/cctx_params.rs index 9bec9b89b..c14184ff9 100644 --- a/c-api/src/cctx_params.rs +++ b/c-api/src/cctx_params.rs @@ -4,7 +4,9 @@ use core::ffi::c_int; -use codec::encoding::{CompressionLevel, CompressionParameters, LiteralCompressionMode, Strategy}; +use codec::encoding::{ + CompressionLevel, CompressionParameters, LiteralCompressionMode, RowMatchFinderMode, Strategy, +}; use crate::cdict::ZSTD_compressionParameters; use crate::context::{ZSTD_CCtx, free_boxed, try_box}; @@ -46,7 +48,7 @@ impl ParamSwitch { /// searches, schedules or checks its work, with no counterpart in this /// encoder: stored as set so they read back, without effect on the frames. /// The sequence-format ones are read by the sequence entry points. -const ADVISORY: [c_int; 18] = [ +const ADVISORY: [c_int; 17] = [ ZSTD_C_NB_WORKERS, ZSTD_C_JOB_SIZE, ZSTD_C_OVERLAP_LOG, @@ -59,7 +61,6 @@ const ADVISORY: [c_int; 18] = [ ZSTD_C_BLOCK_DELIMITERS, ZSTD_C_VALIDATE_SEQUENCES, ZSTD_C_SPLIT_AFTER_SEQUENCES, - ZSTD_C_USE_ROW_MATCH_FINDER, ZSTD_C_DETERMINISTIC_REF_PREFIX, ZSTD_C_PREFETCH_CDICT_TABLES, ZSTD_C_ENABLE_SEQ_PRODUCER_FALLBACK, @@ -105,6 +106,8 @@ pub(crate) struct CCtxParams { pub(crate) magicless: bool, /// `ZSTD_c_literalCompressionMode`. pub(crate) literal_compression: LiteralCompressionMode, + /// `ZSTD_c_useRowMatchFinder`. + pub(crate) row_match_finder: RowMatchFinderMode, /// `ZSTD_c_srcSizeHint`; 0 = none. pub(crate) src_size_hint: c_int, /// The [`ADVISORY`] parameters, in that order. @@ -147,6 +150,7 @@ impl CCtxParams { max_block_size: 0, magicless: false, literal_compression: LiteralCompressionMode::Auto, + row_match_finder: RowMatchFinderMode::Auto, src_size_hint: 0, advisory: [0; ADVISORY.len()], pledged_src_size: CONTENTSIZE_UNKNOWN, @@ -179,7 +183,8 @@ impl CCtxParams { /// (should be unreachable: every knob was bounds-checked on entry). pub(crate) fn resolve(&self) -> Option { let mut builder = CompressionParameters::builder(CompressionLevel::from_level(self.level)) - .literal_compression(self.literal_compression); + .literal_compression(self.literal_compression) + .row_match_finder(self.row_match_finder); if let Some(v) = self.window_log { builder = builder.window_log(v); } @@ -301,6 +306,13 @@ impl CCtxParams { _ => LiteralCompressionMode::Auto, }; } + ZSTD_C_USE_ROW_MATCH_FINDER => { + self.row_match_finder = match ParamSwitch::from_c(within(value)?) { + Some(ParamSwitch::Enable) => RowMatchFinderMode::Enable, + Some(ParamSwitch::Disable) => RowMatchFinderMode::Disable, + _ => RowMatchFinderMode::Auto, + }; + } // Clamped into range as upstream's multi-threaded build stores them // (zstd_compress.c:866-908); frames are still built on the calling // thread, which a worker count only permits, never requires. @@ -371,6 +383,11 @@ impl CCtxParams { LiteralCompressionMode::Enable => ParamSwitch::Enable.to_c(), LiteralCompressionMode::Disable => ParamSwitch::Disable.to_c(), }, + ZSTD_C_USE_ROW_MATCH_FINDER => match self.row_match_finder { + RowMatchFinderMode::Auto => ParamSwitch::Auto.to_c(), + RowMatchFinderMode::Enable => ParamSwitch::Enable.to_c(), + RowMatchFinderMode::Disable => ParamSwitch::Disable.to_c(), + }, _ => match advisory_slot(param) { Some(slot) => self.advisory[slot], None => return Err(ZSTD_ErrorCode::ZSTD_error_parameter_unsupported), diff --git a/c-api/src/tests/cctx_params.rs b/c-api/src/tests/cctx_params.rs index 358cf6563..3bd614b19 100644 --- a/c-api/src/tests/cctx_params.rs +++ b/c-api/src/tests/cctx_params.rs @@ -45,6 +45,7 @@ const FORCE_MAX_WINDOW: c_int = 1000; const FORCE_ATTACH_DICT: c_int = 1001; const LITERAL_COMPRESSION_MODE: c_int = 1002; const SRC_SIZE_HINT: c_int = 1004; +const USE_ROW_MATCH_FINDER: c_int = 1011; const MAX_BLOCK_SIZE: c_int = 1015; const BLOCK_SPLITTER_LEVEL: c_int = 1017; @@ -421,6 +422,60 @@ fn literal_compression_mode_reaches_the_encoder() { assert_eq!(decompress(&raw, text.len()), text); } +/// `ZSTD_c_useRowMatchFinder` picks the greedy / lazy band's finder: `enable` +/// writes what `auto` writes on a source wide enough for rows and `disable` +/// what `auto` writes on one of 16 KiB or less (upstream +/// `ZSTD_resolveRowMatchFinderMode`), the forced choice differs from the +/// window's, and every frame decodes. A value past the switch is refused. +#[test] +fn use_row_match_finder_reaches_the_encoder() { + let text = |lines: u32| -> Vec { + let mut state = 0x9E37_79B9u32; + let mut out = Vec::new(); + for i in 0..lines { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + out.extend_from_slice( + format!( + "{i:06} level=info shard={} took={}ms\n", + state % 13, + state >> 22 + ) + .as_bytes(), + ); + } + out + }; + let frame = |input: &[u8], mode: c_int| { + let cctx = ZSTD_createCCtx(); + unsafe { + assert_eq!(ZSTD_CCtx_setParameter(cctx, LEVEL, 7), 0); + assert_eq!(ZSTD_CCtx_setParameter(cctx, USE_ROW_MATCH_FINDER, mode), 0); + assert_eq!(get_cctx(cctx, USE_ROW_MATCH_FINDER), mode); + } + let out = compress2(cctx, input); + unsafe { ZSTD_freeCCtx(cctx) }; + assert_eq!(decompress(&out, input.len()), input, "mode {mode}"); + out + }; + let (auto, enable, disable) = (0, 1, 2); + let wide = text(6000); + assert_eq!(frame(&wide, enable), frame(&wide, auto)); + assert_ne!(frame(&wide, disable), frame(&wide, auto)); + let narrow = text(200); + assert!(narrow.len() <= 1 << 14); + assert_eq!(frame(&narrow, disable), frame(&narrow, auto)); + assert_ne!(frame(&narrow, enable), frame(&narrow, auto)); + + let cctx = ZSTD_createCCtx(); + unsafe { + assert_eq!( + code(ZSTD_CCtx_setParameter(cctx, USE_ROW_MATCH_FINDER, 3)), + ZSTD_ErrorCode::ZSTD_error_parameter_outOfBound + ); + ZSTD_freeCCtx(cctx); + } +} + /// A size hint stays for every streamed frame and can be cleared again. #[test] fn src_size_hint_round_trips_streamed_frames() { diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index 321f865f7..853db47e0 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -17,7 +17,7 @@ use std::path::{Path, PathBuf}; use structured_zstd::encoding::{ CompressionContext, CompressionLevel, CompressionParameters, LevelParameters, - LiteralCompressionMode, Strategy, StreamingEncoder, + LiteralCompressionMode, RowMatchFinderMode, Strategy, StreamingEncoder, }; /// Error type for the tool: a boxed message, which is all a command-line @@ -341,6 +341,8 @@ struct Options { advanced: AdvancedParams, /// Whether literals are entropy-coded (`--[no-]compress-literals`). literals: LiteralCompressionMode, + /// The greedy / lazy band's finder (`--[no-]row-match-finder`). + row_match_finder: RowMatchFinderMode, /// Reference file for `--patch-from`: raw-content dictionary compression /// with the window sized to the input, so the whole reference is /// reachable. @@ -1498,6 +1500,7 @@ fn parse_args_into( progress: Progress::Auto, advanced: AdvancedParams::default(), literals: LiteralCompressionMode::Auto, + row_match_finder: RowMatchFinderMode::Auto, patch_from: None, show_default_cparams: false, trainer: Trainer::FastCover, @@ -1612,7 +1615,7 @@ fn parse_args_into( return Ok(Parsed::Handled { pause: *pause }); } // Flags that steer HOW the work is done, not what comes out: - // thread counts, IO strategy, matcher hints. We are + // thread counts, IO strategy. We are // single-threaded and pick our own limits, so accepting them // yields the same valid stream. Upstream takes them, so a // script that passes them must not fail here: that is the @@ -1620,14 +1623,8 @@ fn parse_args_into( "single-thread" => opts.threads = Some(0), "adapt" => opts.adapt = true, "rsyncable" => opts.rsyncable = true, - "sparse" - | "no-sparse" - | "asyncio" - | "no-asyncio" - | "mmap-dict" - | "no-mmap-dict" - | "row-match-finder" - | "no-row-match-finder" => {} + "sparse" | "no-sparse" | "asyncio" | "no-asyncio" | "mmap-dict" + | "no-mmap-dict" => {} // The reference prints a trace of its own file-system helpers // by their C names (`UTIL_stat`, `UTIL_isLink`, ...). There is // nothing of ours that output would describe, and taking the @@ -1643,6 +1640,8 @@ fn parse_args_into( "fake-stderr-is-console" => opts.console.stderr = true, "compress-literals" => opts.literals = LiteralCompressionMode::Enable, "no-compress-literals" => opts.literals = LiteralCompressionMode::Disable, + "row-match-finder" => opts.row_match_finder = RowMatchFinderMode::Enable, + "no-row-match-finder" => opts.row_match_finder = RowMatchFinderMode::Disable, _ => { if long == "fast" { // `--fast` is the level -1 alias. @@ -2310,6 +2309,8 @@ Advanced compression options: --no-dictID Don't write `dictID` into the header (dictionary compression only). --[no-]content-size Write the input size into the frame header when it is known. [Default: Write] --[no-]compress-literals Force (un)compressed literals. + --[no-]row-match-finder Explicitly enable/disable the fast, row-based matchfinder for + the 'greedy', 'lazy', and 'lazy2' strategies. --format=zstd Compress files to the `.zst` format. [Default] @@ -2347,8 +2348,7 @@ Environment: ZSTD_CLEVEL sets the default compression level; ZSTD_NBTHREADS is r Accepted for compatibility, with no effect here: -T#/--threads=#, --single-thread, --auto-threads, -B# and --block-size=# when compressing (the job size of a multi-threaded run), --adapt, --rsyncable, --zstd=ovlog=#, --[no-]sparse, ---[no-]asyncio, --[no-]mmap-dict, --[no-]row-match-finder (compression runs -single-threaded). --adapt and --rsyncable are refused beside --single-thread or +--[no-]asyncio, --[no-]mmap-dict (compression runs single-threaded). --adapt and --rsyncable are refused beside --single-thread or -T0 when compressing. Rejected rather than ignored, because they would change the result: --format= @@ -6291,6 +6291,8 @@ struct FrameSettings { advanced: AdvancedParams, /// Whether literals are entropy-coded (`--[no-]compress-literals`). literals: LiteralCompressionMode, + /// The greedy / lazy band's finder (`--[no-]row-match-finder`). + row_match_finder: RowMatchFinderMode, } impl Default for FrameSettings { @@ -6310,6 +6312,7 @@ impl Default for FrameSettings { dict_id_flag: true, advanced: AdvancedParams::default(), literals: LiteralCompressionMode::Auto, + row_match_finder: RowMatchFinderMode::Auto, } } } @@ -6330,21 +6333,26 @@ impl FrameSettings { dict_id_flag: opts.dict_id_flag, advanced: opts.advanced, literals: opts.literals, + row_match_finder: opts.row_match_finder, } } } /// The fine-grained parameters a frame runs under, or `None` when nothing /// overrides the level: `--zstd=` knobs, `--long` (whose window yields to a -/// `--zstd=wlog=`, as the reference command has it), and the literal mode. -/// The long-distance knobs from `--zstd=` reach the encoder only with -/// `--long`, since there they tune a matcher that otherwise does not run. +/// `--zstd=wlog=`, as the reference command has it), and the literal and +/// match-finder modes. The long-distance knobs from `--zstd=` reach the +/// encoder only with `--long`, since there they tune a matcher that otherwise +/// does not run. fn frame_parameters( level: CompressionLevel, settings: &FrameSettings, ) -> Result> { let advanced = settings.advanced; - if !settings.long && advanced.is_default() && settings.literals == LiteralCompressionMode::Auto + if !settings.long + && advanced.is_default() + && settings.literals == LiteralCompressionMode::Auto + && settings.row_match_finder == RowMatchFinderMode::Auto { return Ok(None); } @@ -6388,7 +6396,9 @@ fn frame_parameters( builder = builder.ldm_hash_rate_log(log); } } - builder = builder.literal_compression(settings.literals); + builder = builder + .literal_compression(settings.literals) + .row_match_finder(settings.row_match_finder); builder .build() .map(Some) diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index 236a7e28f..7c499d94f 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -4491,7 +4491,6 @@ fn performance_only_flags_are_accepted() { &["--sparse", "f"][..], &["--no-asyncio", "f"][..], &["--no-mmap-dict", "f"][..], - &["--no-row-match-finder", "f"][..], ] { let parsed = parse(args); assert!( @@ -7472,6 +7471,65 @@ fn literal_compression_flags_reach_the_frame() { assert_eq!(decoded(&fast_coded).unwrap(), payload); } +/// `--[no-]row-match-finder` picks the greedy / lazy band's finder: forcing the +/// hash chain on a source wide enough for rows writes another frame than the +/// level's own, forcing rows writes the level's own, and both decode. +#[test] +fn row_match_finder_flags_reach_the_frame() { + assert_eq!( + parse(&["--no-row-match-finder", "f"]) + .unwrap() + .row_match_finder, + RowMatchFinderMode::Disable + ); + assert_eq!( + parse(&["--row-match-finder", "f"]) + .unwrap() + .row_match_finder, + RowMatchFinderMode::Enable + ); + assert_eq!( + parse(&["f"]).unwrap().row_match_finder, + RowMatchFinderMode::Auto + ); + + let mut payload = Vec::new(); + let mut state = 0x9E37_79B9u32; + for i in 0..6000u32 { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + payload.extend_from_slice( + format!( + "{i:06} level=info shard={} took={}ms\n", + state % 13, + state >> 22 + ) + .as_bytes(), + ); + } + let frame_with = |row_match_finder: RowMatchFinderMode| { + let mut frame = Vec::new(); + compress_stream( + payload.as_slice(), + &mut frame, + &FrameSettings { + level: 7, + pledged_size: Some(payload.len() as u64), + row_match_finder, + ..FrameSettings::default() + }, + &mut no_dict(), + ) + .unwrap(); + frame + }; + let auto = frame_with(RowMatchFinderMode::Auto); + let rows = frame_with(RowMatchFinderMode::Enable); + let chain = frame_with(RowMatchFinderMode::Disable); + assert_eq!(rows, auto); + assert_ne!(chain, auto); + assert_eq!(decoded(&chain).unwrap(), payload); +} + /// The trainer flags take their tuning the way the reference command reads /// it, and the options built from it follow the reference's rules: a value /// the trainer cannot take is refused, `shrink` defaults its bound to 1%, and diff --git a/zstd/src/bit_io/bit_writer.rs b/zstd/src/bit_io/bit_writer.rs index 5e4d32445..9a198e82d 100644 --- a/zstd/src/bit_io/bit_writer.rs +++ b/zstd/src/bit_io/bit_writer.rs @@ -115,143 +115,36 @@ impl>> BitWriter { self.bit_idx += data.len() * 8; } - /// Pre-reserve additional capacity in the output buffer so the - /// upstream zstd-faithful FSE fast path can do `flush_bulk` writes without - /// triggering `Vec::extend_from_slice`'s grow-on-realloc branch. - /// `additional` is the number of bytes the upcoming bursts will - /// emit (caller's bit budget / 8, rounded up). - #[inline] - pub fn reserve_output(&mut self, additional: usize) { - self.output.as_mut().reserve(additional); - } - - /// Upstream zstd `BIT_addBitsFast` (`bitstream.h:193-200`): always accumulate - /// `bits` into the bottom of `partial`, no overflow check. - /// - /// # Safety - /// - /// Caller MUST guarantee BOTH preconditions BEFORE calling: + /// Start a [`BitBurst`] that commits at most `max_bytes` bytes, counting the + /// up to 7 whole bytes it drains from the accumulator here so that it starts + /// with under 8 bits held. /// - /// 1. `num_bits + bits_in_partial <= 64` AND **not both 64**, i.e. - /// if `bits_in_partial == 64` then `num_bits` MUST be 0 — and - /// the explicit `num_bits == 0` early-return below handles that - /// case without ever evaluating `bits << bits_in_partial`. The - /// point of the precondition: any other combination would either - /// shift-by-64 (undefined for `u64 << 64` in Rust) or overflow - /// `bits_in_partial` past 64. - /// 2. `num_bits == 64 || bits >> num_bits == 0` — value must be - /// clean of high junk past `num_bits`. Dirty high bits leak - /// into the packed stream at the next OR. - /// - /// The `debug_assert!`s below catch both in test/debug builds but - /// do NOT survive `cargo build --release`, hence the `unsafe` - /// signature. The function does not perform any memory-unsafe - /// operation, but a violation produces a silently-corrupted - /// output stream that decoders cannot recover — equivalent in - /// blast radius to undefined behaviour for the consumer. - /// - /// Used by the upstream zstd-faithful FSE sequence encoder which knows its - /// per-sequence bit budget at compile time and inserts explicit - /// [`Self::flush_bulk`] calls between bursts — saving the - /// per-call branch + spill that `write_bits_64`'s overflow check - /// pays. - #[inline(always)] - pub unsafe fn write_bits_64_no_check(&mut self, bits: u64, num_bits: usize) { - debug_assert!( - num_bits + self.bits_in_partial <= 64, - "write_bits_64_no_check would overflow partial: would push to {} bits", - num_bits + self.bits_in_partial, - ); - debug_assert!( - self.bits_in_partial < 64 || num_bits == 0, - "write_bits_64_no_check called with full accumulator and num_bits>0; \ - caller must flush_bulk before adding more bits", - ); - debug_assert!( - num_bits == 64 || bits >> num_bits == 0, - "value has dirty high bits beyond num_bits={num_bits}", - ); - // Masked so a full accumulator cannot make this a shift by 64, which is - // undefined for `u64`. It is not a guard costing anything: x86 and - // AArch64 shift instructions mask the count themselves, so the mask - // disappears. It replaces a `num_bits == 0` early return, which was a - // branch on every call, and there are six a sequence. Upstream needs - // neither, because its `BIT_addBitsFast` keeps `bitPos` strictly under - // 64 and can shift unconditionally. - // - // Correct in the one case the mask changes: the accumulator is full - // only where the caller passes `num_bits == 0`, and the second - // precondition then forces `bits == 0`, so the OR contributes nothing - // whatever the shift count. - self.partial |= bits << (self.bits_in_partial & 63); - self.bits_in_partial += num_bits; - } - - /// Upstream zstd `BIT_flushBitsFast` (`bitstream.h:202-214`): write the - /// full 8 bytes of `partial` directly to the output buffer via a - /// single `MEM_writeLEST` (unaligned 8-byte LE store), then - /// advance the Vec's `len` by only `nb_bytes` so the scratch - /// bytes past the commit point get overwritten by the next - /// flush. No overflow check — caller must have pre-reserved at - /// least 8 bytes of spare capacity via [`Self::reserve_output`]. - /// - /// `extend_from_slice` for tiny (0..=7-byte) tails was previously - /// hot — its per-call capacity check + memcpy dispatch cost - /// dwarfs the actual byte write at FSE flush cadence (~54K - /// flushes per compress on the L1 fast path). The direct - /// unaligned store path matches upstream zstd's hot loop cycle-for-cycle. - /// - /// # Safety - /// - /// Caller MUST guarantee that `output.capacity() >= output.len() + - /// 8` BEFORE calling this method (typically via a prior - /// [`Self::reserve_output`]). Violating the precondition produces a - /// 8-byte out-of-bounds write to whatever memory follows the - /// `Vec`'s allocation — undefined behaviour. The accompanying - /// `debug_assert!` catches misuse in test/debug builds but does - /// NOT survive `cargo build --release`, hence the `unsafe` - /// signature. - #[inline(always)] - pub unsafe fn flush_bulk(&mut self) { - let nb_bytes = self.bits_in_partial >> 3; - let bytes = self.partial.to_le_bytes(); + /// For a loop that knows its bit budget per step and flushes between + /// steps itself, which is how upstream writes the sequence section. + pub(crate) fn burst(&mut self, max_bytes: usize) -> BitBurst<'_, V> { + // Room for every flush of the burst: each writes 8 bytes and keeps at + // most 7 of them, so `max_bytes` committed plus 8 past the last. + let room = max_bytes + .checked_add(8) + .expect("a burst larger than the address space"); let output = self.output.as_mut(); - // `bit_idx` counts the bits already committed to `output`, and every - // path that appends bytes advances both, so it is the buffer's length - // in bits at all times. Taking the length from it keeps this off the - // `Vec` header, which is otherwise loaded on every flush, and this runs - // two to three times per encoded sequence. - let len = self.bit_idx >> 3; - debug_assert_eq!( - len, - output.len(), - "bit_idx must track the committed output length", - ); - debug_assert!( - output.capacity() >= len + 8, - "flush_bulk requires 8 bytes of spare capacity; caller forgot reserve_output", - ); - // SAFETY: the function-level Safety contract requires - // `output.capacity() >= len + 8`. We write 8 bytes starting at - // `len`, then commit only `nb_bytes` — the remaining `8 - - // nb_bytes` bytes stay within capacity but past `len`, and - // the next flush_bulk overwrites them. - unsafe { - let dst = output.as_mut_ptr().add(len); - core::ptr::copy_nonoverlapping(bytes.as_ptr(), dst, 8); - output.set_len(len + nb_bytes); - } - // `nb_bytes == 8` means the accumulator was full (64 bits). A - // raw `partial >>= 64` is UB on `u64`, so we zero explicitly. - // Upstream zstd's `BIT_flushBitsFast` writes a fresh 8 bytes the next - // round so the post-flush state must be clean either way. - if nb_bytes == 8 { - self.partial = 0; - } else { - self.partial >>= nb_bytes * 8; - } - self.bits_in_partial &= 7; - self.bit_idx += nb_bytes * 8; + output.reserve(room); + let len = output.len(); + debug_assert_eq!(len * 8, self.bit_idx, "bit_idx tracks the output length"); + let base = output.as_mut_ptr(); + let mut burst = BitBurst { + partial: self.partial, + bits_in_partial: self.bits_in_partial, + // SAFETY: `len <= capacity`. + cursor: unsafe { base.add(len) }, + // SAFETY: `reserve(room)` above made `len + room <= capacity`. + end: unsafe { base.add(len + room) }, + writer: self, + }; + // SAFETY: the accumulator holds under 64 bits outside a burst + // (`write_bits_64` drains it at 64), and the reserve covers 8 bytes. + unsafe { burst.flush() }; + burst } /// Bridge for the upstream zstd-faithful Huffman encoder (`HufCStream`) so @@ -452,5 +345,110 @@ impl>> BitWriter { } } +/// A run of unchecked writes into a [`BitWriter`] with the accumulator and the +/// output cursor held in locals, upstream's `BIT_CStream_t` (`bitstream.h`). +/// +/// Through the writer, every add stored the accumulator back and every flush +/// reloaded the buffer pointer from the `Vec` and stored its length: the byte +/// store goes through a `u8` pointer, which may alias the writer and the `Vec` +/// header, so neither could stay in a register across it. The burst holds no +/// such state behind a reference and commits the length once, in +/// [`Self::finish`]. +#[must_use = "a burst's bits reach the writer only through `finish`"] +pub(crate) struct BitBurst<'a, V: AsMut>> { + writer: &'a mut BitWriter, + partial: u64, + bits_in_partial: usize, + /// Next byte to write. + cursor: *mut u8, + /// End of the room [`BitWriter::burst`] reserved; every flush stays + /// 8 bytes short of it. + end: *mut u8, +} + +impl>> BitBurst<'_, V> { + /// Upstream zstd `BIT_addBitsFast` (`bitstream.h:193-200`): accumulate the + /// low `num_bits` of `bits` above what is held, with no overflow check. + /// + /// # Safety + /// + /// The accumulator must have room: `bits_in_partial + num_bits <= 64`, and + /// `bits` must be clean above `num_bits` (`num_bits == 64 || bits >> + /// num_bits == 0`). Neither violation touches memory, but either corrupts + /// the stream with nothing to detect it, so the caller proves both from + /// its bit budget. Debug builds assert them. + #[inline(always)] + pub(crate) unsafe fn add(&mut self, bits: u64, num_bits: usize) { + debug_assert!( + self.bits_in_partial + num_bits <= 64, + "{num_bits} bits do not fit beside the {} held", + self.bits_in_partial, + ); + debug_assert!( + num_bits == 64 || bits >> num_bits == 0, + "value has dirty high bits beyond num_bits={num_bits}", + ); + // Masked so a full accumulator cannot make this a shift by 64, which is + // undefined for `u64`. x86 and AArch64 shifts mask the count + // themselves, so the mask costs nothing, where a `num_bits == 0` early + // return was a branch on every add. The accumulator is full only when + // `num_bits == 0`, and the second precondition then makes `bits == 0`, + // so the OR adds nothing whatever the count. + self.partial |= bits << (self.bits_in_partial & 63); + self.bits_in_partial += num_bits; + } + + /// Upstream zstd `BIT_flushBitsFast` (`bitstream.h:202-214`): store all + /// 8 bytes of the accumulator at the cursor (one unaligned store) and + /// advance it by the whole bytes only; the rest is rewritten by the next + /// flush. + /// + /// # Safety + /// + /// The burst must still be within the `max_bytes` it was started with, so + /// that the 8-byte store lands inside the reserved room. Debug builds + /// assert it. + #[inline(always)] + pub(crate) unsafe fn flush(&mut self) { + let nb_bytes = self.bits_in_partial >> 3; + debug_assert!( + self.cursor.wrapping_add(8) <= self.end, + "a burst flushed past the room it reserved", + ); + // SAFETY: the cursor is 8 bytes short of the reserved end (above), and + // an unaligned store has no alignment requirement. + unsafe { + self.cursor + .cast::() + .write_unaligned(self.partial.to_le()); + self.cursor = self.cursor.add(nb_bytes); + } + // A full accumulator (64 bits) leaves nothing: shifting a `u64` by 64 + // is undefined, so it is zeroed instead. + self.partial = if nb_bytes == 8 { + 0 + } else { + self.partial >> (nb_bytes * 8) + }; + self.bits_in_partial &= 7; + } + + /// Commit the bytes written and hand the bits still held back to the + /// writer. + pub(crate) fn finish(self) { + let writer = self.writer; + let output = writer.output.as_mut(); + // SAFETY: the cursor started inside this allocation and every flush + // kept it within the reserved room, and the writer is borrowed for the + // whole burst, so the buffer was never reallocated. + let len = unsafe { self.cursor.offset_from(output.as_mut_ptr()) } as usize; + // SAFETY: `[old len, len)` was written by the flushes, within capacity. + unsafe { output.set_len(len) }; + writer.bit_idx = len * 8; + writer.partial = self.partial; + writer.bits_in_partial = self.bits_in_partial; + } +} + #[cfg(test)] mod tests; diff --git a/zstd/src/bit_io/bit_writer/tests.rs b/zstd/src/bit_io/bit_writer/tests.rs index 5f1faeb50..1472cc734 100644 --- a/zstd/src/bit_io/bit_writer/tests.rs +++ b/zstd/src/bit_io/bit_writer/tests.rs @@ -160,6 +160,28 @@ fn add_multiple_aligned() { assert_eq!(vec![0xFF, 0xF0, 0x0F, 0x00], bw.dump()); } +/// A burst flushing a full accumulator writes all eight bytes and holds +/// nothing after, so the next bits land on a clean byte rather than on bits +/// of the flushed value. +#[test] +fn a_burst_flushes_a_full_accumulator_clean() { + let mut bw = BitWriter::new(); + let mut burst = bw.burst(16); + // SAFETY: 64 bits into an empty accumulator, then 8 into an empty one, + // within the 16 bytes reserved. + unsafe { + burst.add(0x0123_4567_89AB_CDEF, 64); + burst.flush(); + burst.add(0xA5, 8); + burst.flush(); + } + burst.finish(); + assert_eq!( + bw.dump(), + vec![0xEF, 0xCD, 0xAB, 0x89, 0x67, 0x45, 0x23, 0x01, 0xA5] + ); +} + // #[test] // fn catches_more_than_in_buf() { // todo!(); diff --git a/zstd/src/decoding/literals_section_decoder/burst_gate_tests.rs b/zstd/src/decoding/literals_section_decoder/burst_gate_tests.rs index dc4d0d63e..9bd7e6e2f 100644 --- a/zstd/src/decoding/literals_section_decoder/burst_gate_tests.rs +++ b/zstd/src/decoding/literals_section_decoder/burst_gate_tests.rs @@ -49,7 +49,10 @@ fn build_huf4x_block(data: &[u8]) -> (LiteralsSection, Vec) { { let mut writer = BitWriter::from(&mut source); let mut encoder = HuffmanEncoder::new(&table, &mut writer); - encoder.encode4x(data, true); + crate::encoding::fastpath::with_encode_tier!( + crate::encoding::fastpath::select_kernel(), + K => encoder.encode4x::(data, true) + ); writer.flush(); } let section = LiteralsSection { diff --git a/zstd/src/dictionary/legacy.rs b/zstd/src/dictionary/legacy.rs index 698be1d5d..30fc0f691 100644 --- a/zstd/src/dictionary/legacy.rs +++ b/zstd/src/dictionary/legacy.rs @@ -13,9 +13,7 @@ use alloc::vec; use alloc::vec::Vec; use super::suffix_array::suffix_array; -use crate::encoding::fastpath::{ - FastpathKernel, dispatch_common_prefix_len_ptr_with_kernel, select_kernel, -}; +use crate::encoding::fastpath::{EncodeTier, select_kernel}; /// Fewest repetitions that make a prefix a candidate (`MINRATIO`). const MIN_RATIO: u32 = 4; @@ -80,19 +78,20 @@ fn noise_band() -> [u8; NOISE_LENGTH] { /// the two. The reference copies the whole corpus into a buffer one band /// longer (`ZDICT_trainFromBuffer_legacy`), a second copy of the samples while /// the caller still holds them for the finalizer. -struct Corpus<'a> { +/// +/// `K` is the compare kernel tier, chosen once for the whole run. +struct Corpus<'a, K: EncodeTier> { samples: &'a [u8], noise: [u8; NOISE_LENGTH], - /// The compare kernel for this CPU, resolved once for the whole run. - kernel: FastpathKernel, + tier: core::marker::PhantomData, } -impl<'a> Corpus<'a> { +impl<'a, K: EncodeTier> Corpus<'a, K> { fn new(samples: &'a [u8]) -> Self { Self { samples, noise: noise_band(), - kernel: select_kernel(), + tier: core::marker::PhantomData, } } @@ -156,14 +155,7 @@ impl<'a> Corpus<'a> { let base = samples.as_ptr(); // SAFETY: `a, b <= max(a, b)` and `in_samples == samples.len() - // max(a, b)`, so both runs of `in_samples` bytes lie in the samples. - let n = unsafe { - dispatch_common_prefix_len_ptr_with_kernel( - self.kernel, - base.add(a), - base.add(b), - in_samples, - ) - }; + let n = unsafe { K::common_prefix_len_ptr(base.add(a), base.add(b), in_samples) }; if n < in_samples { return n; } @@ -279,6 +271,24 @@ pub(crate) fn train_legacy_raw( sample_sizes: &[usize], dict_size: usize, selectivity: u32, +) -> Result, TooSmall> { + train_legacy_raw_under( + select_kernel(), + samples, + sample_sizes, + dict_size, + selectivity, + ) +} + +/// [`train_legacy_raw`] comparing through the tier `tier`, the run's one +/// choice of compare kernel. +fn train_legacy_raw_under( + tier: crate::encoding::fastpath::FastpathKernel, + samples: &[u8], + sample_sizes: &[usize], + dict_size: usize, + selectivity: u32, ) -> Result, TooSmall> { let total: usize = sample_sizes.iter().sum(); debug_assert_eq!(total, samples.len(), "the sizes describe the corpus"); @@ -318,8 +328,23 @@ pub(crate) fn train_legacy_raw( kept -= 1; used -= sample_sizes[kept]; } - let corpus = Corpus::new(&samples[..used]); - find_segments(&mut list, &corpus, used, min_rep); + // The analysis below is compiled for the run's tier. + crate::encoding::fastpath::with_encode_tier!( + tier, + K => select_content::(&mut list, &samples[..used], min_rep, dict_size) + ) +} + +/// Analyse `samples` and assemble the dictionary content from the best +/// segments that fit in `dict_size`, comparing through the tier `K`. +fn select_content( + list: &mut [DictItem], + samples: &[u8], + min_rep: u32, + dict_size: usize, +) -> Result, TooSmall> { + let corpus = Corpus::::new(samples); + find_segments(list, &corpus, samples.len(), min_rep); let segments = list[0].pos as usize; let content_size: usize = list[1..segments].iter().map(|d| d.length as usize).sum(); @@ -353,7 +378,12 @@ pub(crate) fn train_legacy_raw( /// `ZDICT_trainBuffer_legacy`: walk every uncovered position of the corpus in /// text order and insert the segment its suffix neighbourhood yields. -fn find_segments(list: &mut [DictItem], corpus: &Corpus<'_>, len: usize, min_rep: u32) { +fn find_segments( + list: &mut [DictItem], + corpus: &Corpus<'_, K>, + len: usize, + min_rep: u32, +) { let min_ratio = min_rep.max(MIN_RATIO); debug_assert_eq!(corpus.samples.len(), len); let sa = suffix_array(corpus.samples); @@ -410,11 +440,11 @@ fn mark_run(done: &mut [bool], at: usize, len: usize) { /// `ZDICT_analyzePos`: the segment the suffix of rank `start` leads to, or an /// empty one. Every position the analysis settles is marked in `done`. -fn analyze_position( +fn analyze_position( done: &mut [bool], suffixes: &Suffixes, mut start: i64, - corpus: &Corpus<'_>, + corpus: &Corpus<'_, K>, min_ratio: u32, ) -> DictItem { let mut pos = suffixes.at(start); @@ -575,7 +605,7 @@ fn analyze_position( /// Whether the `length` bytes at `a` equal those at `b` (`isIncluded`); both /// runs must lie inside the corpus. Compares `length` bytes and no more, where /// `common` would run on to the end of the shared prefix. -fn is_included(corpus: &Corpus<'_>, a: usize, b: usize, length: usize) -> bool { +fn is_included(corpus: &Corpus<'_, K>, a: usize, b: usize, length: usize) -> bool { let samples = corpus.samples; if let (Some(x), Some(y)) = (samples.get(a..a + length), samples.get(b..b + length)) { return x == y; @@ -597,7 +627,12 @@ fn promote(list: &mut [DictItem], mut at: usize) -> usize { /// `ZDICT_tryMerge`: fold `elt` into an entry it overlaps, skipping entry /// `skip`. Returns the merged entry's index, or 0 when nothing merged. -fn try_merge(list: &mut [DictItem], elt: DictItem, skip: usize, corpus: &Corpus<'_>) -> usize { +fn try_merge( + list: &mut [DictItem], + elt: DictItem, + skip: usize, + corpus: &Corpus<'_, K>, +) -> usize { let size = list[0].pos as usize; let elt_end = elt.pos + elt.length; @@ -682,7 +717,12 @@ fn remove_item(list: &mut [DictItem], id: usize) { /// `ZDICT_insertDictItem`: merge `elt` into the table if it overlaps an entry, /// and keep merging while the merged entry overlaps another; otherwise insert /// it in savings order, dropping the last entry when the table is full. -fn insert_item(list: &mut [DictItem], max_size: u32, elt: DictItem, corpus: &Corpus<'_>) { +fn insert_item( + list: &mut [DictItem], + max_size: u32, + elt: DictItem, + corpus: &Corpus<'_, K>, +) { let mut merge_id = try_merge(list, elt, 0, corpus); if merge_id != 0 { loop { diff --git a/zstd/src/dictionary/legacy/tests.rs b/zstd/src/dictionary/legacy/tests.rs index 8bf76c46e..78c91a4a3 100644 --- a/zstd/src/dictionary/legacy/tests.rs +++ b/zstd/src/dictionary/legacy/tests.rs @@ -12,7 +12,7 @@ fn neighbour_walks_stop_at_the_noise_slots() { // The corpus is the band itself, and its one suffix is the only rank in // the array: every walk leaves the array on its first step. let band = noise_band(); - let corpus = Corpus::new(&band); + let corpus = Corpus::::new(&band); let suffixes = Suffixes::new(vec![0], band.len()); let mut done = vec![false; band.len() + 16]; let solution = analyze_position(&mut done, &suffixes, 0, &corpus, MIN_RATIO); @@ -33,7 +33,7 @@ fn the_measuring_walk_stops_at_the_noise_slot() { .cycle() .take(4 * NOISE_LENGTH) .collect(); - let corpus = Corpus::new(&corpus_bytes); + let corpus = Corpus::::new(&corpus_bytes); // In suffix order: each copy's suffix is a prefix of the one before it. let suffixes = Suffixes::new(vec![96, 64, 32, 0], corpus_bytes.len()); let mut done = vec![false; corpus_bytes.len() + 16]; @@ -85,6 +85,18 @@ fn the_content_is_made_of_repeated_corpus_runs() { assert!(!text.contains(&format!("ts={:08}", 3 * 7919))); } +/// Every tier the CPU can run trains the same content: the run compares +/// through one tier, and the scalar fallback must find what the SIMD compares +/// find. +#[test] +fn every_tier_trains_the_same_content() { + let (samples, sizes) = log_samples(400); + let content = crate::encoding::fastpath::same_under_every_tier(|tier| { + train_legacy_raw_under(tier, &samples, &sizes, 4096, 0) + }); + assert!(content.is_ok_and(|content| !content.is_empty())); +} + /// A size below the reference's minimum, a corpus under 512 bytes, and a corpus /// with nothing repeated are refused, as `ZDICT_trainFromBuffer_legacy` refuses /// them. diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index 262648996..7a4d76af6 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -6,6 +6,7 @@ use crate::{ bit_io::BitWriter, blocks::block::BlockType, encoding::block_header::BlockHeader, + encoding::fastpath::EncodeTier, encoding::frame_compressor::{CompressState, FseTables, PreviousFseTable, SharedFseTable}, encoding::workspace::{RegionVec, Workspace, region_bytes}, encoding::{Matcher, Sequence}, @@ -430,13 +431,13 @@ impl Matcher for EntropyOnlyMatcher { /// returned for the caller to [`commit_block_tables`] if it keeps the block, /// or [`discard_block_tables`] if it writes the block raw instead. #[must_use = "the block's tables have to be committed or discarded"] -pub(crate) fn compress_block( +pub(crate) fn compress_block( state: &mut CompressState, output: &mut Vec, ) -> BlockTables { let mut scratch = core::mem::take(&mut state.block_scratch); - collect_block_parts(state, &mut scratch.parts); - let tables = encode_block_parts( + collect_block_parts::(state, &mut scratch.parts); + let tables = encode_block_parts::( state, &scratch.parts.literals, &mut scratch.parts.sequences, @@ -447,7 +448,7 @@ pub(crate) fn compress_block( tables } -pub(crate) fn compress_block_with_post_split( +pub(crate) fn compress_block_with_post_split( state: &mut CompressState, last_block: bool, output: &mut Vec, @@ -455,7 +456,7 @@ pub(crate) fn compress_block_with_post_split( #[cfg(all(feature = "lsm", feature = "hash"))] mut block_checksums: Option<&mut Vec>, ) { let mut scratch = core::mem::take(&mut state.block_scratch); - collect_block_parts(state, &mut scratch.parts); + collect_block_parts::(state, &mut scratch.parts); if scratch.parts.sequences.len() <= 4 { let source_len = state.matcher.get_last_space().len(); #[cfg(feature = "lsm")] @@ -480,7 +481,7 @@ pub(crate) fn compress_block_with_post_split( compressed: &mut scratch.compressed, codes: &mut scratch.sequence_codes, }; - let emitted_raw = emit_single_sequence_block( + let emitted_raw = emit_single_sequence_block::( state, last_block, source_len, @@ -625,7 +626,7 @@ pub(crate) fn compress_block_with_post_split( compressed: &mut scratch.compressed, codes: &mut scratch.sequence_codes, }; - let emitted_raw = emit_single_sequence_block( + let emitted_raw = emit_single_sequence_block::( state, last_block && last_partition, src_size, @@ -653,7 +654,10 @@ pub(crate) fn compress_block_with_post_split( /// sequence (`ZSTD_storeSeq`); here the match loop calls the emit from a dozen /// sites, and a copy inlined at each of them costs the loop more in decode /// pressure than the calls it saves (+2.16% cycles at level 3, measured). -fn collect_block_parts(state: &mut CompressState, parts: &mut EncodedBlockParts) { +fn collect_block_parts( + state: &mut CompressState, + parts: &mut EncodedBlockParts, +) { parts.literals.clear(); parts.sequences.clear(); // Trailing literals, reported after the last sequence; the runs are read @@ -679,9 +683,17 @@ fn collect_block_parts(state: &mut CompressState, parts: &mut Enc let code_offsets = uses_fast_offset_codes(state.strategy_tag); let block_start_hist = state.offset_hist; let mut hist = state.offset_hist; - state.matcher.start_matching(|seq| { - record_sequence(seq, &mut tail, sequences, code_offsets, &mut hist); - }); + // The built-in matcher searches compiled for the frame's tier; a matcher + // from outside the crate knows nothing of tiers. The branch folds per + // matcher type. + match state.matcher.builtin_driver() { + Some(driver) => { + builtin_block_sequences::(driver, &mut tail, sequences, code_offsets, &mut hist) + } + None => state.matcher.start_matching(|seq| { + record_sequence(seq, &mut tail, sequences, code_offsets, &mut hist) + }), + } if code_offsets { state.offset_hist = hist; } @@ -698,9 +710,7 @@ fn collect_block_parts(state: &mut CompressState, parts: &mut Enc block_start_hist, ); } - let kernel = state.copy_kernel; - literal_runs::gather_literals( - kernel, + literal_runs::gather_literals::( state.matcher.get_last_space(), &parts.sequences, tail, @@ -708,6 +718,23 @@ fn collect_block_parts(state: &mut CompressState, parts: &mut Enc ); } +/// The built-in matcher's search of the block under tier `K`, recorded as in +/// [`collect_block_parts`]. Generic over the tier alone: the record closure's +/// type carries every parameter of the function it is written in, so written +/// in `collect_block_parts` it would compile the whole search again for each +/// matcher type that wraps the driver. +#[inline] +fn builtin_block_sequences( + driver: &mut crate::encoding::match_generator::MatchGeneratorDriver, + tail: &mut usize, + sequences: &mut RegionVec, + code_offsets: bool, + hist: &mut [u32; 3], +) { + driver + .start_matching_with::(|seq| record_sequence(seq, tail, sequences, code_offsets, hist)); +} + /// Append a block's parse to `capture` as external sequences with plain /// offsets and their repeat-offset numbers, against the `history` the block /// started from, closed by a delimiter carrying the `tail` literals. With @@ -786,7 +813,7 @@ fn record_sequence( } } -fn encode_block_parts( +fn encode_block_parts( state: &mut CompressState, literals_vec: &[u8], raw_sequences: &mut [RawSequence], @@ -854,7 +881,7 @@ fn encode_block_parts( rle_literals(literals_vec, &mut writer); HuffOutcome::Clear } else if literals_vec.len() >= min_lits { - match compress_literals( + match compress_literals::( literals_vec, state.last_huff_table.as_ref(), &mut writer, @@ -974,13 +1001,11 @@ fn encode_block_parts( encode_table(&of_mode, &mut writer); encode_table(&ml_mode, &mut writer); - encode_sequences( + encode_sequences::( raw_sequences, codes, &mut writer, - &ll_mode, - &ml_mode, - &of_mode, + [&ll_mode, &ml_mode, &of_mode], [ll_default, ml_default, of_default], ); @@ -1745,7 +1770,7 @@ struct SingleSequenceEmitBuffers<'a> { codes: &'a mut RegionVec, } -fn emit_single_sequence_block( +fn emit_single_sequence_block( state: &mut CompressState, last_block: bool, source_len: usize, @@ -1757,7 +1782,7 @@ fn emit_single_sequence_block( // The tables are only read here and the block's decisions come back to be // committed, so writing the partition raw leaves nothing to restore. buffers.compressed.clear(); - let tables = encode_block_parts( + let tables = encode_block_parts::( state, literals, sequences, @@ -2776,243 +2801,289 @@ fn commit_last_used_table( } } -fn encode_sequences( - sequences: &[RawSequence], - // One packed [`SequenceCode`] per sequence, in the same order, from the - // pass that derived the offset codes. - codes: &[u32], - writer: &mut BitWriter<&mut Vec>, - ll_mode: &FseTableMode<'_>, - ml_mode: &FseTableMode<'_>, - of_mode: &FseTableMode<'_>, - // The three predefined tables, in LL / ML / OF order. Passed rather than - // read off `FseTables`, whose accessors borrow the whole struct while the - // caller is holding its slots. - defaults: [&FSETable; 3], -) { - fn mode_table<'a>(mode: &'a FseTableMode<'_>, default: &'a FSETable) -> Option<&'a FSETable> { - mode.as_table(default) - } - - debug_assert_eq!(codes.len(), sequences.len()); - let sequence = sequences[sequences.len() - 1]; - let code = SequenceCode(codes[codes.len() - 1]); - let (ll_code, ll_num_bits) = (code.ll_code(), code.ll_bits()); - let (ml_code, ml_num_bits) = (code.ml_code(), code.ml_bits()); - let (of_code, of_num_bits) = (code.of_code(), code.of_bits()); - let ll_add_bits = low_bits(sequence.ll, ll_num_bits); - let ml_add_bits = low_bits(sequence.ml - 3, ml_num_bits); - let of_add_bits = low_bits(sequence.off_base, of_num_bits); - let [ll_default, ml_default, of_default] = defaults; - let ll_table = mode_table(ll_mode, ll_default); - let ml_table = mode_table(ml_mode, ml_default); - let of_table = mode_table(of_mode, of_default); - // Carried as bare indices, not `Option`s. A component has a state exactly - // when it has a table, and the tables are resolved once above the loop, so - // wrapping the state too meant re-asking the same question three times per - // sequence and re-wrapping the answer. An absent component's index is never - // read: every site that touches one is already inside its table's `Some`. - let mut ll_state = ll_table.map_or(0, |table| table.start_state(ll_code).index); - let mut ml_state = ml_table.map_or(0, |table| table.start_state(ml_code).index); - let mut of_state = of_table.map_or(0, |table| table.start_state(of_code).index); - - writer.write_bits(ll_add_bits, ll_num_bits); - writer.write_bits(ml_add_bits, ml_num_bits); - writer.write_bits(of_add_bits, of_num_bits); - - // Upstream zstd-faithful sequence loop: write state diffs + extras via - // unchecked fast-path adds with explicit `flush_bulk` calls at - // safe burst boundaries. Per-sequence bit budget: - // state diffs: of (<=8) + ml (<=9) + ll (<=9) = 26 bits. - // extras: ll (<=16) + ml (<=16) + of (<=31). - // - // One flush per sequence is the common case: the two inside the body are - // conditional on the extras not fitting beside the diffs, which is - // upstream's own accounting in `ZSTD_encodeSequences_body` - // (`zstd_compress_sequences.c:303-367`), and the one at the end of the - // body is what leaves the accumulator with under a byte for the next - // round. - // - // Pre-reserve output capacity for the worst-case sequence section - // size (~10 bytes/sequence + 32 byte slack) so the per-flush - // `extend_from_slice` never triggers a Vec realloc. - if sequences.len() > 1 { - writer.reserve_output(sequences.len() * 12 + 64); - // Pre-loop flush: the safe `write_bits` calls above for the - // final sequence's add_bits leave `bits_in_partial` in - // 0..=63. Before the first unchecked-add burst we drain to - // < 8 leftover so the per-burst budget math (state diffs ≤ - // 30 + leftover ≤ 8 = 38 < 64) holds invariantly. - // SAFETY: `reserve_output` above guarantees capacity ≥ - // current_len + sequences.len() * 12 + 64 ≥ current_len + 8. - unsafe { - writer.flush_bulk(); - } - // Walked as a slice rather than by index: the same order, without the - // bounds check and the index arithmetic that `sequences[i]` pays on - // every sequence. The last one is coded through the FSE init states - // above, so it is not in this range. - for (&sequence, &code) in sequences[..sequences.len() - 1] - .iter() - .zip(codes[..codes.len() - 1].iter()) - .rev() - { - let code = SequenceCode(code); - let (ll_code, ll_num_bits) = (code.ll_code(), code.ll_bits()); - let (ml_code, ml_num_bits) = (code.ml_code(), code.ml_bits()); - let (of_code, of_num_bits) = (code.of_code(), code.of_bits()); - let ll_add_bits = low_bits(sequence.ll, ll_num_bits); - let ml_add_bits = low_bits(sequence.ml - 3, ml_num_bits); - let of_add_bits = low_bits(sequence.off_base, of_num_bits); - - // State diffs burst: max 30 bits (10+10+9 worst case for - // acc_log ≤ 9 ll/ml + acc_log ≤ 8 of) + ≤ 7 leftover from - // prior flush = ≤ 37 bits total — well under 64. - // - // SAFETY (for every `write_bits_64_no_check` below): - // - the prior `flush_bulk` left `bits_in_partial ≤ 7`; - // - each FSE state diff has `next.num_bits ≤ acc_log ≤ 10`; - // three diffs back-to-back add ≤ 30 bits → total ≤ 37, - // well below the 64-bit accumulator cap. - // - `diff = state.index - next.baseline` cannot exceed - // `(1 << num_bits) - 1`, so `diff >> num_bits == 0`. - // `reserve_output(sequences.len() * 12 + 64)` above - // pre-allocated enough spare capacity to cover every - // per-sequence flush in this loop (≤ 16 bytes per - // sequence, plus the 32-byte slack on top of the 64-byte - // header reserve). - // - // What the three diffs actually put in the accumulator, tallied so - // the budget below is checked against the widths the FSE tables - // produced rather than against the ceiling its derivation assumed. - #[cfg(debug_assertions)] - let mut state_diff_bits = 0usize; - if let Some(table) = of_table { - let next = table.next_state(of_code, of_state); - let diff = crate::fse::fse_encoder::transition_bits(of_state, next.num_bits); - unsafe { - writer.write_bits_64_no_check(diff as u64, next.num_bits as usize); - } +/// The sequence section's bitstream, shared by [`encode_sequences_scalar`] and +/// `encode_sequences_bmi2`. A macro so each tier's loop is compiled inside its +/// own function and the bit writer and the FSE lookups inline into it. +macro_rules! encode_sequences_body { + ($sequences:expr, $codes:expr, $writer:expr, $modes:expr, $defaults:expr) => {{ + let sequences: &[RawSequence] = $sequences; + let codes: &[u32] = $codes; + let writer: &mut BitWriter<&mut Vec> = $writer; + let [ll_mode, ml_mode, of_mode]: [&FseTableMode<'_>; 3] = $modes; + let [ll_default, ml_default, of_default]: [&FSETable; 3] = $defaults; + + debug_assert_eq!(codes.len(), sequences.len()); + let sequence = sequences[sequences.len() - 1]; + let code = SequenceCode(codes[codes.len() - 1]); + let (ll_code, ll_num_bits) = (code.ll_code(), code.ll_bits()); + let (ml_code, ml_num_bits) = (code.ml_code(), code.ml_bits()); + let (of_code, of_num_bits) = (code.of_code(), code.of_bits()); + let ll_add_bits = low_bits(sequence.ll, ll_num_bits); + let ml_add_bits = low_bits(sequence.ml - 3, ml_num_bits); + let of_add_bits = low_bits(sequence.off_base, of_num_bits); + let ll_table = ll_mode.as_table(ll_default); + let ml_table = ml_mode.as_table(ml_default); + let of_table = of_mode.as_table(of_default); + // Carried as bare indices, not `Option`s. A component has a state + // exactly when it has a table, and the tables are resolved once above + // the loop, so wrapping the state too meant re-asking the same question + // three times per sequence and re-wrapping the answer. An absent + // component's index is never read: every site that touches one is + // already inside its table's `Some`. + let mut ll_state = ll_table.map_or(0, |table| table.start_state(ll_code).index); + let mut ml_state = ml_table.map_or(0, |table| table.start_state(ml_code).index); + let mut of_state = of_table.map_or(0, |table| table.start_state(of_code).index); + + writer.write_bits(ll_add_bits, ll_num_bits); + writer.write_bits(ml_add_bits, ml_num_bits); + writer.write_bits(of_add_bits, of_num_bits); + + // Upstream zstd-faithful sequence loop: write state diffs + extras via + // unchecked adds with explicit flushes at safe boundaries, through a + // burst that keeps the accumulator and the cursor in registers. + // Per-sequence bit budget: + // state diffs: of (<=8) + ml (<=9) + ll (<=9) = 26 bits. + // extras: ll (<=16) + ml (<=16) + of (<=31). + // + // One flush per sequence is the common case: the two inside the body + // are conditional on the extras not fitting beside the diffs, which is + // upstream's own accounting in `ZSTD_encodeSequences_body` + // (`zstd_compress_sequences.c:303-367`), and the one at the end of the + // body is what leaves the accumulator with under a byte for the next + // round. + if sequences.len() > 1 { + // At most 63 bits a sequence leave as whole bytes, under 8 bytes; + // 12 a sequence and 64 over covers that and the up to 7 bytes the + // burst drains from the init-state writes above as it starts, + // which also leaves it holding under 8 bits. + let mut burst = writer.burst(sequences.len() * 12 + 64); + // Walked as a slice rather than by index: the same order, without + // the bounds check and the index arithmetic that `sequences[i]` + // pays on every sequence. The last one is coded through the FSE + // init states above, so it is not in this range. + for (&sequence, &code) in sequences[..sequences.len() - 1] + .iter() + .zip(codes[..codes.len() - 1].iter()) + .rev() + { + let code = SequenceCode(code); + let (ll_code, ll_num_bits) = (code.ll_code(), code.ll_bits()); + let (ml_code, ml_num_bits) = (code.ml_code(), code.ml_bits()); + let (of_code, of_num_bits) = (code.of_code(), code.of_bits()); + let ll_add_bits = low_bits(sequence.ll, ll_num_bits); + let ml_add_bits = low_bits(sequence.ml - 3, ml_num_bits); + let of_add_bits = low_bits(sequence.off_base, of_num_bits); + + // State diffs burst: max 30 bits (10+10+9 worst case for + // acc_log ≤ 9 ll/ml + acc_log ≤ 8 of) + ≤ 7 leftover from + // prior flush = ≤ 37 bits total — well under 64. + // + // SAFETY (for every `burst.add` below): + // - the prior flush left `bits_in_partial ≤ 7`; + // - each FSE state diff has `next.num_bits ≤ acc_log ≤ 10`; + // three diffs back-to-back add ≤ 30 bits → total ≤ 37, + // well below the 64-bit accumulator cap. + // - `diff = state.index - next.baseline` cannot exceed + // `(1 << num_bits) - 1`, so `diff >> num_bits == 0`. + // And for every `burst.flush`: the burst was started with room + // for 12 bytes a sequence, more than one sequence's flushes + // commit. + // + // What the three diffs actually put in the accumulator, tallied + // so the budget below is checked against the widths the FSE + // tables produced rather than against the ceiling its + // derivation assumed. #[cfg(debug_assertions)] - { - state_diff_bits += next.num_bits as usize; + let mut state_diff_bits = 0usize; + if let Some(table) = of_table { + let next = table.next_state(of_code, of_state); + let diff = crate::fse::fse_encoder::transition_bits(of_state, next.num_bits); + unsafe { + burst.add(diff as u64, next.num_bits as usize); + } + #[cfg(debug_assertions)] + { + state_diff_bits += next.num_bits as usize; + } + of_state = next.index; } - of_state = next.index; - } - if let Some(table) = ml_table { - let next = table.next_state(ml_code, ml_state); - let diff = crate::fse::fse_encoder::transition_bits(ml_state, next.num_bits); - unsafe { - writer.write_bits_64_no_check(diff as u64, next.num_bits as usize); + if let Some(table) = ml_table { + let next = table.next_state(ml_code, ml_state); + let diff = crate::fse::fse_encoder::transition_bits(ml_state, next.num_bits); + unsafe { + burst.add(diff as u64, next.num_bits as usize); + } + #[cfg(debug_assertions)] + { + state_diff_bits += next.num_bits as usize; + } + ml_state = next.index; + } + if let Some(table) = ll_table { + let next = table.next_state(ll_code, ll_state); + let diff = crate::fse::fse_encoder::transition_bits(ll_state, next.num_bits); + unsafe { + burst.add(diff as u64, next.num_bits as usize); + } + #[cfg(debug_assertions)] + { + state_diff_bits += next.num_bits as usize; + } + ll_state = next.index; } + // The three state diffs and the three extra-bit fields share + // one 64-bit accumulator, and a flush here is only needed when + // the extras that follow would not fit beside them. Upstream + // asks exactly that question (`zstd_compress_sequences.c:350`): + // `ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + + // OffFSELog)`, which with our accumulator logs (9 + 9 + 8 = 26) + // is 31. Below it the accumulator holds at most 7 leftover + 26 + // diff bits + 30 extra bits = 63, so nothing has to leave yet. + // Flushing unconditionally instead cost a second store, length + // commit and shift on every sequence, and a level-1 block has + // hundreds of thousands of them. + let extra_bits_total = of_num_bits + ml_num_bits + ll_num_bits; + // The thresholds are arithmetic on widths, so they are only + // right while the widths are what they were derived from. + // Pinned here, where the arithmetic happens, so a widened + // encoder fails at its cause rather than as an accumulator + // overflow further down or, in a release build, as a corrupted + // stream with nothing to point at. #[cfg(debug_assertions)] { - state_diff_bits += next.num_bits as usize; + debug_assert!( + state_diff_bits <= 26, + "state diffs took {state_diff_bits} bits; the thresholds below are \ + derived from a 26-bit ceiling (LLFSELog 9 + MLFSELog 9 + OffFSELog 8)", + ); + debug_assert!( + ll_num_bits <= 16 && ml_num_bits <= 16 && of_num_bits <= 31, + "extra-bit widths ll={ll_num_bits} ml={ml_num_bits} of={of_num_bits} \ + exceed the 16 / 16 / 31 the thresholds are derived from", + ); + // The bound the branch itself rests on: without a flush, + // the leftover, the diffs and all three extra fields have + // to fit. + debug_assert!( + extra_bits_total >= 31 || 7 + state_diff_bits + extra_bits_total <= 64, + "no flush at {extra_bits_total} extra bits, but 7 leftover + \ + {state_diff_bits} diff bits + {extra_bits_total} would overflow the \ + 64-bit accumulator", + ); } - ml_state = next.index; - } - if let Some(table) = ll_table { - let next = table.next_state(ll_code, ll_state); - let diff = crate::fse::fse_encoder::transition_bits(ll_state, next.num_bits); + if extra_bits_total >= 31 { + unsafe { + burst.flush(); + } + } + + // Extras burst: ll (≤16) + ml (≤16) + of (≤ 31). Whether the + // accumulator was drained above or not, ll and ml fit: after a + // flush it holds ≤ 7 + 32 = 39, and without one the three + // extras together were under 31. + // + // SAFETY: `encode_literal_length` / `encode_match_len` + // bound `*_num_bits ≤ 16` and return a clean `*_add_bits` + // (low `num_bits` bits only). `encode_offset` bounds + // `of_num_bits ≤ ilog2(of) ≤ 31`, and the two conditionals + // (`>= 31` above, `> 56` below) are upstream's own accounting + // for when the offset field still fits beside what is already + // there. unsafe { - writer.write_bits_64_no_check(diff as u64, next.num_bits as usize); + burst.add(ll_add_bits as u64, ll_num_bits); + burst.add(ml_add_bits as u64, ml_num_bits); } - #[cfg(debug_assertions)] - { - state_diff_bits += next.num_bits as usize; + // Upstream `zstd_compress_sequences.c:355`: with the + // accumulator drained before the diffs, 7 leftover plus all + // three extra fields must stay under 64, so the offset needs + // its own container only past 56 bits of extras. + if extra_bits_total > 56 { + unsafe { + burst.flush(); + } } - ll_state = next.index; - } - // The three state diffs and the three extra-bit fields share one - // 64-bit accumulator, and a flush here is only needed when the - // extras that follow would not fit beside them. Upstream asks - // exactly that question (`zstd_compress_sequences.c:350`): - // `ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + - // OffFSELog)`, which with our accumulator logs (9 + 9 + 8 = 26) is - // 31. Below it the accumulator holds at most 7 leftover + 26 diff - // bits + 30 extra bits = 63, so nothing has to leave yet. Flushing - // unconditionally instead cost a second store, length commit and - // shift on every sequence, and a level-1 block has hundreds of - // thousands of them. - let extra_bits_total = of_num_bits + ml_num_bits + ll_num_bits; - // The thresholds are arithmetic on widths, so they are only right - // while the widths are what they were derived from. Pinned here, - // where the arithmetic happens, so a widened encoder fails at its - // cause rather than as an accumulator overflow further down or, in - // a release build, as a corrupted stream with nothing to point at. - #[cfg(debug_assertions)] - { - debug_assert!( - state_diff_bits <= 26, - "state diffs took {state_diff_bits} bits; the thresholds below are \ - derived from a 26-bit ceiling (LLFSELog 9 + MLFSELog 9 + OffFSELog 8)", - ); - debug_assert!( - ll_num_bits <= 16 && ml_num_bits <= 16 && of_num_bits <= 31, - "extra-bit widths ll={ll_num_bits} ml={ml_num_bits} of={of_num_bits} \ - exceed the 16 / 16 / 31 the thresholds are derived from", - ); - // The bound the branch itself rests on: without a flush, the - // leftover, the diffs and all three extra fields have to fit. - debug_assert!( - extra_bits_total >= 31 || 7 + state_diff_bits + extra_bits_total <= 64, - "no flush at {extra_bits_total} extra bits, but 7 leftover + \ - {state_diff_bits} diff bits + {extra_bits_total} would overflow the \ - 64-bit accumulator", - ); - } - if extra_bits_total >= 31 { unsafe { - writer.flush_bulk(); + burst.add(of_add_bits as u64, of_num_bits); + burst.flush(); } } + burst.finish(); + } + if let Some(table) = ml_table { + writer.write_bits(ml_state as u64, table.table_size.ilog2() as usize); + } + if let Some(table) = of_table { + writer.write_bits(of_state as u64, table.table_size.ilog2() as usize); + } + if let Some(table) = ll_table { + writer.write_bits(ll_state as u64, table.table_size.ilog2() as usize); + } - // Extras burst: ll (≤16) + ml (≤16) + of (≤ 31). Whether the - // accumulator was drained above or not, ll and ml fit: after a - // flush it holds ≤ 7 + 32 = 39, and without one the three extras - // together were under 31. - // - // SAFETY: `encode_literal_length` / `encode_match_len` - // bound `*_num_bits ≤ 16` and return a clean `*_add_bits` - // (low `num_bits` bits only). `encode_offset` bounds - // `of_num_bits ≤ ilog2(of) ≤ 31`, and the two conditionals - // (`>= 31` above, `> 56` below) are upstream's own accounting for - // when the offset field still fits beside what is already there. - unsafe { - writer.write_bits_64_no_check(ll_add_bits as u64, ll_num_bits); - writer.write_bits_64_no_check(ml_add_bits as u64, ml_num_bits); - } - // Upstream `zstd_compress_sequences.c:355`: with the accumulator - // drained before the diffs, 7 leftover plus all three extra fields - // must stay under 64, so the offset needs its own container only - // past 56 bits of extras. - if extra_bits_total > 56 { - unsafe { - writer.flush_bulk(); - } - } - unsafe { - writer.write_bits_64_no_check(of_add_bits as u64, of_num_bits); - writer.flush_bulk(); - } + let bits_to_fill = writer.misaligned(); + if bits_to_fill == 0 { + writer.write_bits(1u32, 8); + } else { + writer.write_bits(1u32, bits_to_fill); } - } - if let Some(table) = ml_table { - writer.write_bits(ml_state as u64, table.table_size.ilog2() as usize); - } - if let Some(table) = of_table { - writer.write_bits(of_state as u64, table.table_size.ilog2() as usize); - } - if let Some(table) = ll_table { - writer.write_bits(ll_state as u64, table.table_size.ilog2() as usize); - } + }}; +} - let bits_to_fill = writer.misaligned(); - if bits_to_fill == 0 { - writer.write_bits(1u32, 8); - } else { - writer.write_bits(1u32, bits_to_fill); - } +/// Write the sequence section's bitstream for `sequences` under the three +/// table `modes`, in LL / ML / OF order. +/// +/// Where the tier `K` has BMI2 the loop runs under it, as upstream's does +/// (`ZSTD_encodeSequences_bmi2`): every field is written at a width known only +/// at run time, and BMI2's `shlx` / `shrx` / `bzhi` take that width in any +/// register, where the base x86 shifts take it only in `CL` and stall on the +/// flags they leave. The bytes written are the same on every tier. +fn encode_sequences( + sequences: &[RawSequence], + // One packed [`SequenceCode`] per sequence, in the same order, from the + // pass that derived the offset codes. + codes: &[u32], + writer: &mut BitWriter<&mut Vec>, + modes: [&FseTableMode<'_>; 3], + // The three predefined tables, in the same order as `modes`. Passed rather + // than read off `FseTables`, whose accessors borrow the whole struct while + // the caller is holding its slots. + defaults: [&FSETable; 3], +) { + crate::encoding::fastpath::tier_match!(K { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] + // SAFETY: the tier is only selected after confirming BMI2 (and AVX2). + crate::encoding::fastpath::FastpathKernel::Avx2Bmi2 => unsafe { + encode_sequences_bmi2(sequences, codes, writer, modes, defaults) + }, + _ => encode_sequences_scalar(sequences, codes, writer, modes, defaults), + }) +} + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" +))] +#[target_feature(enable = "bmi2")] +unsafe fn encode_sequences_bmi2( + sequences: &[RawSequence], + codes: &[u32], + writer: &mut BitWriter<&mut Vec>, + modes: [&FseTableMode<'_>; 3], + defaults: [&FSETable; 3], +) { + encode_sequences_body!(sequences, codes, writer, modes, defaults) +} + +fn encode_sequences_scalar( + sequences: &[RawSequence], + codes: &[u32], + writer: &mut BitWriter<&mut Vec>, + modes: [&FseTableMode<'_>; 3], + defaults: [&FSETable; 3], +) { + encode_sequences_body!(sequences, codes, writer, modes, defaults) } fn encode_seqnum(seqnum: usize, writer: &mut BitWriter>>) { @@ -3288,7 +3359,7 @@ fn rle_literals(literals: &[u8], writer: &mut BitWriter<&mut Vec>) { /// `compress_literals` byte-for-byte (same size_format ladder, same /// min_gain raw-fallback gate) so the wire output is identical to /// the size-comparison reuse path when both would pick reuse. -fn emit_reuse_literals( +fn emit_reuse_literals( literals: &[u8], last_table: &huff0_encoder::HuffmanTable, writer: &mut BitWriter<&mut Vec>, @@ -3315,7 +3386,7 @@ fn emit_reuse_literals( if size_format == 0 { encoder.encode(literals, false); } else { - encoder.encode4x(literals, false); + encoder.encode4x::(literals, false); } let encoded_len = (writer.index() - index_before) / 8; writer.change_bits(size_index, encoded_len as u64, size_bits); @@ -3385,7 +3456,7 @@ fn end_samples_look_flat(literals: &[u8], counts: &mut [usize; 256]) -> bool { head_largest + tail_largest <= ((2 * SUSPECT_SAMPLE_SIZE) >> 7) + 4 } -fn compress_literals( +fn compress_literals( literals: &[u8], last_table: Option<&huff0_encoder::HuffmanTable>, writer: &mut BitWriter<&mut Vec>, @@ -3410,7 +3481,7 @@ fn compress_literals( && let Some(prev) = last_table && prev.estimate_compressed_size(literals).is_some() { - return emit_reuse_literals(literals, prev, writer, reset_idx, strategy); + return emit_reuse_literals::(literals, prev, writer, reset_idx, strategy); } let mut counts = [0usize; 256]; @@ -3508,7 +3579,7 @@ fn compress_literals( if size_format == 0 { encoder.encode(literals, new_table) } else { - encoder.encode4x(literals, new_table) + encoder.encode4x::(literals, new_table) }; let encoded_len = (writer.index() - index_before) / 8; writer.change_bits(size_index, encoded_len as u64, size_bits); diff --git a/zstd/src/encoding/blocks/compressed/literal_runs.rs b/zstd/src/encoding/blocks/compressed/literal_runs.rs index f03c50708..aacd0d858 100644 --- a/zstd/src/encoding/blocks/compressed/literal_runs.rs +++ b/zstd/src/encoding/blocks/compressed/literal_runs.rs @@ -7,7 +7,7 @@ //! block, so no copy below the dispatch asks which kernel to use. use crate::decoding::simd_copy; -use crate::encoding::fastpath::FastpathKernel; +use crate::encoding::fastpath::EncodeTier; use crate::encoding::workspace::RegionVec; use super::RawSequence; @@ -18,16 +18,12 @@ use super::RawSequence; const LITERAL_INLINE_COPY_MAX: usize = 2048; /// Append the literal runs of `sequences`, then `tail` trailing literal bytes, -/// from `block` to `dst`. -/// -/// `kernel` is the tier resolved when the compressor was built; this is the -/// only place it is looked at, once per block. +/// from `block` to `dst`, with the copies of the tier `K`. /// /// # Panics /// If the sequences and the tail reach past `block`, which a matcher that /// reports its sequences correctly never produces. -pub(super) fn gather_literals( - kernel: FastpathKernel, +pub(super) fn gather_literals( block: &[u8], sequences: &[RawSequence], tail: usize, @@ -39,19 +35,22 @@ pub(super) fn gather_literals( if dst.capacity() - dst.len() < block.len() { crate::encoding::workspace::capacity_exhausted(); } - match kernel { + crate::encoding::fastpath::tier_match!(K { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-avx2" ))] // SAFETY: the tier is only selected after confirming AVX2 (and BMI2). - FastpathKernel::Avx2Bmi2 => unsafe { gather_avx2(block, sequences, tail, dst) }, + crate::encoding::fastpath::FastpathKernel::Avx2Bmi2 => unsafe { + gather_avx2(block, sequences, tail, dst) + }, #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-sse" ))] // SAFETY: both tiers are only selected after confirming SSE2. - FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { + crate::encoding::fastpath::FastpathKernel::Sse2 + | crate::encoding::fastpath::FastpathKernel::Sse42 => unsafe { gather_sse2(block, sequences, tail, dst) }, #[cfg(all( @@ -59,15 +58,17 @@ pub(super) fn gather_literals( target_endian = "little", feature = "kernel-neon" ))] - FastpathKernel::Neon => gather_neon(block, sequences, tail, dst), + crate::encoding::fastpath::FastpathKernel::Neon => gather_neon(block, sequences, tail, dst), #[cfg(all( target_arch = "wasm32", target_feature = "simd128", feature = "kernel-simd128" ))] - FastpathKernel::Simd128 => gather_simd128(block, sequences, tail, dst), + crate::encoding::fastpath::FastpathKernel::Simd128 => { + gather_simd128(block, sequences, tail, dst) + }, _ => gather_scalar(block, sequences, tail, dst), - } + }) } /// Bytes of the block that must remain past a run for it to be copied in whole diff --git a/zstd/src/encoding/blocks/compressed/tests.rs b/zstd/src/encoding/blocks/compressed/tests.rs index 901e25626..aa83bb16e 100644 --- a/zstd/src/encoding/blocks/compressed/tests.rs +++ b/zstd/src/encoding/blocks/compressed/tests.rs @@ -424,12 +424,15 @@ fn estimator_literals_section_mirrors_emit_for_short_inputs() { let est = estimate_block_parts_size(&mut est_state, &literals, &[], &mut workspace); let mut emitted: Vec = Vec::new(); let mut codes_ws = Workspace::new(); - encode_block_parts( - &mut emit_state, - &literals, - &mut [], - &mut code_buffer(&mut codes_ws, 0), - &mut emitted, + crate::encoding::fastpath::with_encode_tier!( + emit_state.copy_kernel, + K => encode_block_parts::<_, K>( + &mut emit_state, + &literals, + &mut [], + &mut code_buffer(&mut codes_ws, 0), + &mut emitted, + ) ); assert_eq!( est, @@ -493,12 +496,15 @@ fn a_section_with_flat_ends_costs_what_the_emitter_writes_for_it() { let est = estimate_block_parts_size(&mut est_state, &literals, &[], &mut workspace); let mut emitted: Vec = Vec::new(); let mut codes_ws = Workspace::new(); - encode_block_parts( - &mut emit_state, - &literals, - &mut [], - &mut code_buffer(&mut codes_ws, 0), - &mut emitted, + crate::encoding::fastpath::with_encode_tier!( + emit_state.copy_kernel, + K => encode_block_parts::<_, K>( + &mut emit_state, + &literals, + &mut [], + &mut code_buffer(&mut codes_ws, 0), + &mut emitted, + ) ); assert_eq!( @@ -795,12 +801,15 @@ fn estimator_and_emitter_agree_on_a_block_with_sequences() { let mut to_emit = sequences.clone(); let mut codes_ws = Workspace::new(); let mut codes = code_buffer(&mut codes_ws, to_emit.len()); - encode_block_parts( - &mut emit_state, - &literals, - &mut to_emit, - &mut codes, - &mut emitted, + crate::encoding::fastpath::with_encode_tier!( + emit_state.copy_kernel, + K => encode_block_parts::<_, K>( + &mut emit_state, + &literals, + &mut to_emit, + &mut codes, + &mut emitted, + ) ); // Both sides must have advanced the repeat-offset history the same @@ -870,13 +879,16 @@ fn raw_partition_fallback_restores_repeat_offset_history() { compressed: &mut compressed_scratch, codes: &mut code_scratch, }; - let emitted_raw = emit_single_sequence_block( - &mut state, - true, - source.len(), - &[], - &mut sequences, - &mut emit_buffers, + let emitted_raw = crate::encoding::fastpath::with_encode_tier!( + state.copy_kernel, + K => emit_single_sequence_block::<_, K>( + &mut state, + true, + source.len(), + &[], + &mut sequences, + &mut emit_buffers, + ) ); if emitted_raw { output.extend_from_slice(&source); @@ -1206,8 +1218,7 @@ fn literals_buffer(capacity: usize) -> (Workspace, RegionVec) { #[should_panic(expected = "workspace buffer sized below what the frame put in it")] fn literals_past_the_buffer_stop_instead_of_overflowing() { let (_ws, mut literals) = literals_buffer(4); - super::literal_runs::gather_literals( - crate::encoding::fastpath::FastpathKernel::Scalar, + super::literal_runs::gather_literals::( &[0xA5; 8], &[], 8, @@ -1227,8 +1238,7 @@ fn sequences_past_the_block_are_refused() { ml: 60, off_base: 1, }]; - super::literal_runs::gather_literals( - crate::encoding::fastpath::FastpathKernel::Scalar, + super::literal_runs::gather_literals::( &[0x5A; 32], &sequences, 0, @@ -1306,11 +1316,96 @@ fn gathered_literals_are_the_runs_in_order() { expected.extend_from_slice(&block[pos..]); for kernel in runnable_fastpath_kernels() { let (_ws, mut literals) = literals_buffer(block_len); - super::literal_runs::gather_literals(kernel, &block, &sequences, 13, &mut literals); + crate::encoding::fastpath::with_encode_tier!( + kernel, + K => super::literal_runs::gather_literals::(&block, &sequences, 13, &mut literals) + ); assert_eq!(&literals[..], &expected[..], "{kernel:?}"); } } +/// Every tier this CPU can run writes the same block, byte for byte, as the +/// scalar tier: the tiers differ in the instructions of the Huffman burst and +/// the sequence-section loop (BMI2 against base shifts), never in their output. +/// The block carries a Huffman-coded literals section and thousands of +/// sequences, so both loops run on every tier. +#[test] +fn every_tier_writes_the_block_the_scalar_tier_writes() { + use super::{CompressedBlockScratch, EntropyOnlyMatcher, encode_block_parts}; + // Skewed literals, so the section is Huffman-coded rather than raw. + let literals: Vec = (0..24_000u32) + .map(|i| b"eeeettaoinshrdlucmfwyp"[(i.wrapping_mul(2654435761) >> 27) as usize % 22]) + .collect(); + // Lengths and offsets across their extra-bit ranges, so the field widths + // the loop shifts by vary from one sequence to the next. + let mut sequences: Vec = (0..4_000u32) + .map(|i| { + let mix = i.wrapping_mul(0x9E37_79B9); + RawSequence { + ll: (mix >> 7) % 70, + ml: 3 + (mix >> 13) % 300, + off_base: 1 + (mix >> 3) % 100_000, + } + }) + .collect(); + // Wide extras for both conditional flushes of the loop: 16 + 16 + 24 = + // 56 extra bits takes the one before the extras (31 or more) alone, and + // 16 + 16 + 30 = 62 takes the one before the offset (more than 56) too. + for (k, seq) in sequences.iter_mut().step_by(97).enumerate() { + // The first values with 16 extra bits: literal length 65536 and match + // length 65539 (`ml` counts the 3 the match-length code leaves out). + seq.ll = 65_536 + k as u32; + seq.ml = 65_539 + k as u32; + seq.off_base = if k % 2 == 0 { + (1u32 << 24) + k as u32 + } else { + (1u32 << 30) + k as u32 + }; + } + let encode = |kernel| { + let mut state = CompressState:: { + matcher: EntropyOnlyMatcher, + sequence_capture: None, + copy_kernel: kernel, + last_huff_table: None, + huff_table_spare: None, + huff_weights: Default::default(), + seen_content: Default::default(), + fse_tables: FseTables::new(), + block_scratch: CompressedBlockScratch::new(), + workspace: Workspace::new(), + offset_hist: [1, 4, 8], + strategy_tag: StrategyTag::Lazy, + pre_split: None, + huf_optimal_search: true, + literal_compression_disabled: false, + post_split_allowed: true, + }; + let mut emitted = Vec::new(); + let mut to_emit = sequences.clone(); + let mut codes_ws = Workspace::new(); + let mut codes = code_buffer(&mut codes_ws, to_emit.len()); + crate::encoding::fastpath::with_encode_tier!( + kernel, + K => encode_block_parts::<_, K>( + &mut state, + &literals, + &mut to_emit, + &mut codes, + &mut emitted, + ) + ); + emitted + }; + let scalar = encode(crate::encoding::fastpath::FastpathKernel::Scalar); + // The literals section is Huffman-coded (Literals_Block_Type 2 in the low + // two bits of its first byte, RFC 8878 3.1.1.3.1.1), so the burst ran. + assert_eq!(scalar[0] & 0b11, 2, "literals section is not Huffman-coded"); + for kernel in runnable_fastpath_kernels() { + assert_eq!(encode(kernel), scalar, "{kernel:?}"); + } +} + #[test] fn choose_table_without_previous_does_not_unwrap_none() { let only_zero_one_table = build_table_from_symbol_counts(&[1, 1], 5, false); diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs index 6774f6282..8250fea0d 100644 --- a/zstd/src/encoding/bt/mod.rs +++ b/zstd/src/encoding/bt/mod.rs @@ -27,9 +27,7 @@ use super::opt::types::{HcOptimalNode, HcOptimalPlanBuffers, HcOptimalSequence, /// Maximum offset reachable by the HC3 short-match probe. Upstream zstd /// parity: keeps the 3-byte side table from emitting offsets that -/// the main BT/HC paths would address more efficiently. Used inside -/// the `hash3_candidate_body!` macro that the kernel-specific -/// variants below expand. +/// the main BT/HC paths would address more efficiently. pub(crate) const HC3_MAX_OFFSET: usize = 1 << 18; /// Binary-tree matcher state used by the `BtOpt` / `BtUltra` / diff --git a/zstd/src/encoding/cparams.rs b/zstd/src/encoding/cparams.rs index 40c52c4a1..5e11f7460 100644 --- a/zstd/src/encoding/cparams.rs +++ b/zstd/src/encoding/cparams.rs @@ -328,11 +328,15 @@ pub(crate) fn copy_cparams(cdict: CParams, window_log: u32) -> CParams { cp } -/// `ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, cp)`: the greedy / lazy / -/// lazy2 strategies search rows only above a 2^14 window. A CDict resolves -/// this from its own cParams and the frame inherits the answer. -pub(crate) fn uses_row_match_finder(cp: &CParams) -> bool { - (3..=5).contains(&cp.strategy) && cp.window_log > 14 +/// `ZSTD_resolveRowMatchFinderMode(mode, cp)` + `ZSTD_rowMatchFinderUsed` +/// (zstd_compress.c:240-261): whether the greedy / lazy / lazy2 strategies +/// search rows. A CDict resolves this from its own cParams and mode, and the +/// frame inherits the answer. +pub(crate) fn uses_row_match_finder( + cp: &CParams, + mode: crate::encoding::parameters::RowMatchFinderMode, +) -> bool { + (3..=5).contains(&cp.strategy) && mode.searches_rows(1usize << cp.window_log) } /// `ZSTD_shouldAttachDict`: the CDict tables are referenced in place diff --git a/zstd/src/encoding/dfast/extend_with_repcode_tests.rs b/zstd/src/encoding/dfast/extend_with_repcode_tests.rs index 7360071f7..643c87f8d 100644 --- a/zstd/src/encoding/dfast/extend_with_repcode_tests.rs +++ b/zstd/src/encoding/dfast/extend_with_repcode_tests.rs @@ -10,6 +10,7 @@ use alloc::vec; use alloc::vec::Vec; use super::*; +use crate::encoding::fastpath::with_host_tier; /// Capture every sequence the matcher emits into an owned record, so the /// assertions can match on `lit_len` / `offset` / `match_len` shape directly. @@ -81,13 +82,13 @@ fn dfast_repcode_extension_emits_zero_literal_rep_on_constant_run() { let mut seqs = Vec::new(); let new_pos = { let mut rec = record_seq(&mut seqs); - dfast.extend_with_repcode_after_match( + with_host_tier!(K => dfast.extend_with_repcode_after_match::( current_abs_start, current_len, pos, &mut literals_start, &mut rec, - ) + )) }; assert!( @@ -153,13 +154,13 @@ fn dfast_repcode_extension_walks_into_retained_history() { let mut seqs = Vec::new(); let new_pos = { let mut rec = record_seq(&mut seqs); - dfast.extend_with_repcode_after_match( + with_host_tier!(K => dfast.extend_with_repcode_after_match::( current_abs_start, current_len, pos, &mut literals_start, &mut rec, - ) + )) }; assert!( @@ -226,13 +227,13 @@ fn dfast_repcode_extension_accepts_exactly_four_byte_rep() { let mut seqs = Vec::new(); let new_pos = { let mut rec = record_seq(&mut seqs); - dfast.extend_with_repcode_after_match( + with_host_tier!(K => dfast.extend_with_repcode_after_match::( current_abs_start, current_len, pos, &mut literals_start, &mut rec, - ) + )) }; // Helper must emit a single 4-byte rep, then stop because @@ -371,12 +372,12 @@ fn dfast_borrowed_window_matches_owned_sequence_stream() { owned.commit_input(&whole[..split]); { let mut rec = record_seq(&mut owned_seqs); - owned.start_matching(&mut rec); + with_host_tier!(K => owned.start_matching::(&mut rec)); } owned.commit_input(&whole[split..]); { let mut rec = record_seq(&mut owned_seqs); - owned.start_matching(&mut rec); + with_host_tier!(K => owned.start_matching::(&mut rec)); } // Borrowed: same bytes, scanned in place by absolute block range. @@ -388,11 +389,11 @@ fn dfast_borrowed_window_matches_owned_sequence_stream() { unsafe { borrowed.set_borrowed_window(&whole) }; { let mut rec = record_seq(&mut borrowed_seqs); - borrowed.start_matching_borrowed(0, split, &mut rec); + with_host_tier!(K => borrowed.start_matching_borrowed::(0, split, &mut rec)); } { let mut rec = record_seq(&mut borrowed_seqs); - borrowed.start_matching_borrowed(split, whole.len(), &mut rec); + with_host_tier!(K => borrowed.start_matching_borrowed::(split, whole.len(), &mut rec)); } assert_eq!( diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index e8ff8c2c2..916ad77c0 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -16,7 +16,7 @@ use core::convert::TryInto; use super::Sequence; use super::blocks::encode_offset_with_history; use super::dict_attach::{DictAttach, DictKeep}; -use super::fastpath::{FastpathKernel, select_kernel}; +use super::fastpath::EncodeTier; use super::incompressible::block_looks_incompressible; use super::levels::config::MIN_WINDOW_LOG; use super::match_generator::{ @@ -24,7 +24,7 @@ use super::match_generator::{ DFAST_REBASE_GUARD_BAND, DFAST_SHORT_HASH_BITS_DELTA, DFAST_SHORT_HASH_LOOKAHEAD, DFAST_SKIP_STEP_GROWTH_INTERVAL, }; -use super::match_table::helpers::{common_prefix_len_with_kernel, extend_backwards_shared}; +use super::match_table::helpers::extend_backwards_shared; use super::match_table::storage::{REBASE_RESET_FLOOR_CEILING, check_stream_abs_headroom}; use super::opt::types::MatchCandidate; use super::util::hash_top_bits; @@ -174,10 +174,6 @@ pub(crate) struct DfastMatchGenerator { /// (the CDict's `hashLog` / `chainLog`), set by the driver; `None` uses /// the live widths. dict_table_bits: Option<(usize, usize)>, - /// CPU kernel for the `common_prefix_len` / repcode byte-compares, - /// resolved once at construction so the per-byte match-finder scans skip - /// the `select_kernel()` `OnceLock` atomic on every call. - pub(crate) kernel: FastpathKernel, /// Borrowed one-shot input window, registered once per frame: /// `(input_base_ptr, total_len)`. When `Some`, the fast loop sources /// bytes from the caller's in-place input slice with absolute input @@ -330,7 +326,6 @@ impl DfastMatchGenerator { #[cfg(test)] dict_table_builds: (0, 0), dict_table_bits: None, - kernel: select_kernel(), borrowed_input: None, borrowed_block: None, dict_resident: false, @@ -1248,7 +1243,10 @@ impl DfastMatchGenerator { } } - pub(crate) fn start_matching(&mut self, mut handle_sequence: impl FnMut(Sequence)) { + pub(crate) fn start_matching( + &mut self, + mut handle_sequence: impl FnMut(Sequence), + ) { self.ensure_hash_tables(); let current_len = self.window_blocks.back().copied().unwrap_or(0); @@ -1272,7 +1270,7 @@ impl DfastMatchGenerator { // dfast is the upstream zstd's greedy double-fast at every level (no lazy // variant exists — lazy parsing is the separate `ZSTD_lazy`/`lazy2` // strategy), so there is a single match path. - self.start_matching_fast_loop(current_abs_start, current_len, &mut handle_sequence); + self.start_matching_fast_loop::(current_abs_start, current_len, &mut handle_sequence); } /// Register the caller's in-place input buffer for the borrowed one-shot @@ -1327,7 +1325,7 @@ impl DfastMatchGenerator { /// offsets, candidate reads land in the same buffer, and the seam re-seed /// re-hashes the prior block's short-key tail exactly as the owned loop /// does once `history` spans it. - pub(crate) fn start_matching_borrowed( + pub(crate) fn start_matching_borrowed( &mut self, block_start: usize, block_end: usize, @@ -1350,7 +1348,7 @@ impl DfastMatchGenerator { if backfill_start < current_abs_start { self.insert_positions(backfill_start, current_abs_start); } - self.start_matching_fast_loop(current_abs_start, current_len, &mut handle_sequence); + self.start_matching_fast_loop::(current_abs_start, current_len, &mut handle_sequence); } /// Borrowed one-shot equivalent of [`Self::skip_matching`]: stage the @@ -1423,7 +1421,7 @@ impl DfastMatchGenerator { /// updated by this helper — the caller routes through /// `emit_candidate` which inserts via `insert_positions` over the /// emitted range, exactly like the inner loop's hit path. - fn probe_tail_ip0_only( + fn probe_tail_ip0_only( &self, current_abs_start: usize, current_len: usize, @@ -1533,9 +1531,7 @@ impl DfastMatchGenerator { unsafe { let lhs = history_base_ptr.add(history_start_offset + cand_idx + 8); let rhs = history_base_ptr.add(history_start_offset + concat_idx0 + 8); - let ext = crate::encoding::fastpath::dispatch_common_prefix_len_ptr( - lhs, rhs, max_fwd, - ); + let ext = K::common_prefix_len_ptr(lhs, rhs, max_fwd); match_len += ext; } let concat = unsafe { @@ -1573,9 +1569,7 @@ impl DfastMatchGenerator { unsafe { let lhs = history_base_ptr.add(history_start_offset + cand_idx_s + 4); let rhs = history_base_ptr.add(history_start_offset + concat_idx0 + 4); - let ext = crate::encoding::fastpath::dispatch_common_prefix_len_ptr( - lhs, rhs, max_fwd, - ); + let ext = K::common_prefix_len_ptr(lhs, rhs, max_fwd); s_match_len += ext; } let concat = unsafe { @@ -1623,7 +1617,7 @@ impl DfastMatchGenerator { /// rep is a net win over re-running the full hash search. Returns /// the new value of `pos` and updates `literals_start` in place to /// the post-rep-chain anchor. - fn extend_with_repcode_after_match( + fn extend_with_repcode_after_match( &mut self, current_abs_start: usize, current_len: usize, @@ -1688,8 +1682,15 @@ impl DfastMatchGenerator { if !gate_eq { break; } - let match_len = - common_prefix_len_with_kernel(self.kernel, &concat[cand_idx..], &concat[cur_idx..]); + // SAFETY: `cand_idx < cur_idx < concat.len()` (above), and the + // compare stops at the shorter of the two tails. + let match_len = unsafe { + K::common_prefix_len_ptr( + concat.as_ptr().add(cand_idx), + concat.as_ptr().add(cur_idx), + concat.len() - cur_idx, + ) + }; if match_len < DFAST_REP_MIN_MATCH_LEN { break; } @@ -2368,7 +2369,7 @@ fn short_hash_key(data: &[u8]) -> u64 { /// cpl calls inline under one umbrella and `select_kernel()` is resolved ONCE /// per block in the bare dispatcher, never per cpl call. macro_rules! start_matching_fast_loop_body { - ($self:ident, $current_abs_start:ident, $current_len:ident, $handle_sequence:ident, $cpl:path, $borrowed:expr, $tagged:expr) => {{ + ($self:ident, $current_abs_start:ident, $current_len:ident, $handle_sequence:ident, $cpl:path, $tier:ty, $borrowed:expr, $tagged:expr) => {{ // Behaviour change vs the pre-refactor `start_matching_general`: // this fast loop deliberately drops the strict-incompressible // early-skip path (the `block_looks_incompressible_strict` short @@ -2520,7 +2521,7 @@ macro_rules! start_matching_fast_loop_body { // they're skipped — upstream zstd accepts that exact tradeoff at // the iend boundary). if ip1 + HASH_READ_SIZE > $current_len { - if let Some(committed) = $self.probe_tail_ip0_only( + if let Some(committed) = $self.probe_tail_ip0_only::<$tier>( $current_abs_start, $current_len, ip0, @@ -2535,7 +2536,7 @@ macro_rules! start_matching_fast_loop_body { $handle_sequence, ); pos = start + committed.match_len; - pos = $self.extend_with_repcode_after_match( + pos = $self.extend_with_repcode_after_match::<$tier>( $current_abs_start, $current_len, pos, @@ -3272,7 +3273,7 @@ macro_rules! start_matching_fast_loop_body { $handle_sequence, ); pos = start + candidate.match_len; - pos = $self.extend_with_repcode_after_match( + pos = $self.extend_with_repcode_after_match::<$tier>( $current_abs_start, $current_len, pos, @@ -3336,7 +3337,7 @@ macro_rules! start_matching_fast_loop_body { /// slightly FEWER than it does — what is left there is execution density, not /// work. macro_rules! start_matching_dict_loop_body { - ($self:ident, $current_abs_start:ident, $current_len:ident, $handle_sequence:ident, $cpl:path) => {{ + ($self:ident, $current_abs_start:ident, $current_len:ident, $handle_sequence:ident, $cpl:path, $tier:ty) => {{ debug_assert!($current_len > 0, "dict_loop precondition: $current_len > 0"); $self.ensure_room_for($current_abs_start + $current_len - 1); const PRIME: u64 = 0xCF1BBCDCB7A56463_u64; @@ -3758,7 +3759,7 @@ macro_rules! start_matching_dict_loop_body { $handle_sequence, ); pos = start + candidate.match_len; - pos = $self.extend_with_repcode_after_match( + pos = $self.extend_with_repcode_after_match::<$tier>( $current_abs_start, $current_len, pos, @@ -3788,13 +3789,12 @@ enum DfastInnerExit { } impl DfastMatchGenerator { - /// Dispatcher for the per-kernel dfast fast loop: resolve the tier ONCE - /// per block via `select_kernel()` and call the matching - /// `start_matching_fast_loop_` wrapper, so every per-position + /// The dfast loop of the encoder tier `K`: its + /// `start_matching_fast_loop_` wrapper, so every per-position /// `common_prefix_len_ptr` inlines under one `#[target_feature]` umbrella - /// (mirrors `build_optimal_plan_impl`). Replaces the prior per-cpl - /// `dispatch_common_prefix_len_ptr` runtime dispatch. - fn start_matching_fast_loop( + /// (mirrors `build_optimal_plan_impl`). The tier match keeps only that one + /// call. + fn start_matching_fast_loop( &mut self, current_abs_start: usize, current_len: usize, @@ -3867,7 +3867,9 @@ impl DfastMatchGenerator { // target can check the two against each other — the branch is // `cfg(test)` and does not exist in a shipped build. #[cfg(test)] - if use_dict && self.kernel == crate::encoding::fastpath::FastpathKernel::Scalar { + if use_dict + && const { matches!(K::TIER, crate::encoding::fastpath::FastpathKernel::Scalar) } + { return self.start_matching_dict_loop_scalar( current_abs_start, current_len, @@ -3879,10 +3881,7 @@ impl DfastMatchGenerator { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - // Use the matcher-level cached kernel (resolved once in `new()`), - // not a per-block `select_kernel()` — the cache only helps if the - // hot path actually reads it. Mirrors the Row backend. - match self.kernel { + crate::encoding::fastpath::tier_match!(K { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { dispatch_dict!( @@ -3898,7 +3897,7 @@ impl DfastMatchGenerator { start_matching_fast_loop_scalar, start_matching_dict_loop_scalar ), - } + }) } #[cfg(all( target_arch = "wasm32", @@ -3910,7 +3909,9 @@ impl DfastMatchGenerator { // compile time here, so without it this target could never run the // scalar dictionary loop to check the SIMD one against. #[cfg(test)] - if use_dict && self.kernel == crate::encoding::fastpath::FastpathKernel::Scalar { + if use_dict + && const { matches!(K::TIER, crate::encoding::fastpath::FastpathKernel::Scalar) } + { return self.start_matching_dict_loop_scalar( current_abs_start, current_len, @@ -3964,6 +3965,7 @@ impl DfastMatchGenerator { current_len, handle_sequence, crate::encoding::fastpath::neon::common_prefix_len_ptr, + crate::encoding::fastpath::NeonTier, BORROWED, TAGGED ) @@ -3986,6 +3988,7 @@ impl DfastMatchGenerator { current_len, handle_sequence, crate::encoding::fastpath::sse2::common_prefix_len_ptr, + crate::encoding::fastpath::Sse2Tier, BORROWED, TAGGED ) @@ -4008,6 +4011,7 @@ impl DfastMatchGenerator { current_len, handle_sequence, crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, + crate::encoding::fastpath::Avx2Bmi2Tier, BORROWED, TAGGED ) @@ -4031,6 +4035,7 @@ impl DfastMatchGenerator { current_len, handle_sequence, crate::encoding::fastpath::simd128::common_prefix_len_ptr, + crate::encoding::fastpath::Simd128Tier, BORROWED, TAGGED ) @@ -4061,6 +4066,7 @@ impl DfastMatchGenerator { current_len, handle_sequence, crate::encoding::fastpath::scalar::common_prefix_len_ptr, + crate::encoding::fastpath::ScalarTier, BORROWED, TAGGED ) @@ -4083,7 +4089,8 @@ impl DfastMatchGenerator { current_abs_start, current_len, handle_sequence, - crate::encoding::fastpath::neon::common_prefix_len_ptr + crate::encoding::fastpath::neon::common_prefix_len_ptr, + crate::encoding::fastpath::NeonTier ) } @@ -4103,7 +4110,8 @@ impl DfastMatchGenerator { current_abs_start, current_len, handle_sequence, - crate::encoding::fastpath::sse2::common_prefix_len_ptr + crate::encoding::fastpath::sse2::common_prefix_len_ptr, + crate::encoding::fastpath::Sse2Tier ) } @@ -4123,7 +4131,8 @@ impl DfastMatchGenerator { current_abs_start, current_len, handle_sequence, - crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr + crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, + crate::encoding::fastpath::Avx2Bmi2Tier ) } @@ -4144,7 +4153,8 @@ impl DfastMatchGenerator { current_abs_start, current_len, handle_sequence, - crate::encoding::fastpath::simd128::common_prefix_len_ptr + crate::encoding::fastpath::simd128::common_prefix_len_ptr, + crate::encoding::fastpath::Simd128Tier ) } @@ -4180,7 +4190,8 @@ impl DfastMatchGenerator { current_abs_start, current_len, handle_sequence, - crate::encoding::fastpath::scalar::common_prefix_len_ptr + crate::encoding::fastpath::scalar::common_prefix_len_ptr, + crate::encoding::fastpath::ScalarTier ) } } diff --git a/zstd/src/encoding/fastpath/mod.rs b/zstd/src/encoding/fastpath/mod.rs index 3c76f3985..a42bc25b7 100644 --- a/zstd/src/encoding/fastpath/mod.rs +++ b/zstd/src/encoding/fastpath/mod.rs @@ -308,67 +308,267 @@ fn detect_kernel_uncached() -> FastpathKernel { FastpathKernel::Scalar } -/// Public entry point for raw-pointer prefix-length scans (BT byte compare, -/// repcode extend, etc.): resolves the tier via [`select_kernel`] on every -/// call, so hot loops should cache the kernel and call -/// [`dispatch_common_prefix_len_ptr_with_kernel`] instead. -/// -/// # Safety -/// `lhs` / `rhs` must each point to at least `max` initialized bytes. -#[inline] -pub(crate) unsafe fn dispatch_common_prefix_len_ptr( - lhs: *const u8, - rhs: *const u8, - max: usize, -) -> usize { - // Cold-path shim: resolves the kernel via `select_kernel()` on every call. - // Hot match-finder loops resolve the kernel once per block and call - // [`dispatch_common_prefix_len_ptr_with_kernel`] directly. - unsafe { dispatch_common_prefix_len_ptr_with_kernel(select_kernel(), lhs, rhs, max) } +/// A [`FastpathKernel`] tier as a type. The encoder chooses its tier once, in +/// [`with_encode_tier`], and everything below that choice is compiled per tier: +/// a leaf that runs a different kernel per tier picks it with [`tier_match`], +/// which keeps the one kernel at monomorphisation, so no tier is asked at run +/// time below the choice. +pub(crate) trait EncodeTier { + /// The tier this type stands for. + const TIER: FastpathKernel; + + /// Length of the common prefix of `lhs` and `rhs`, at most `max`, with this + /// tier's compare. + /// + /// # Safety + /// `lhs` / `rhs` must each point to at least `max` initialized bytes, and + /// the CPU must support the tier, which holds for the tier the encoder + /// chose. + unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize; +} + +/// One tier type: the struct, its [`FastpathKernel`] and its compare kernel, +/// under the cfg that builds the tier. +macro_rules! encode_tier { + ($(#[$cfg:meta])* $name:ident, $tier:ident, $prefix:path) => { + #[doc = concat!("[`FastpathKernel::", stringify!($tier), "`] as a type.")] + $(#[$cfg])* + pub(crate) struct $name; + $(#[$cfg])* + impl EncodeTier for $name { + const TIER: FastpathKernel = FastpathKernel::$tier; + + #[inline(always)] + unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize { + // SAFETY: forwarded from the caller. + unsafe { $prefix(lhs, rhs, max) } + } + } + }; } -/// Prefix-length scan against an already-resolved [`FastpathKernel`], so a hot -/// loop pays the kernel-select once per block (caller-cached) instead of the -/// `OnceLock` atomic + branch on every byte-compare. +encode_tier!(ScalarTier, Scalar, scalar::common_prefix_len_ptr); +encode_tier!( + #[cfg(all( + target_arch = "aarch64", + target_endian = "little", + feature = "kernel-neon" + ))] + NeonTier, + Neon, + neon::common_prefix_len_ptr +); +// Both SSE tiers compare with the same SSE2 kernel; only the optimal parser's +// price set differs between them. +encode_tier!( + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" + ))] + Sse2Tier, + Sse2, + sse2::common_prefix_len_ptr +); +encode_tier!( + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" + ))] + Sse42Tier, + Sse42, + sse2::common_prefix_len_ptr +); +encode_tier!( + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] + Avx2Bmi2Tier, + Avx2Bmi2, + avx2_bmi2::common_prefix_len_ptr +); +encode_tier!( + #[cfg(all( + target_arch = "wasm32", + target_feature = "simd128", + feature = "kernel-simd128" + ))] + Simd128Tier, + Simd128, + simd128::common_prefix_len_ptr +); + +/// The encoder's one choice of tier: evaluate `$body` with `$k` naming the +/// [`EncodeTier`] type of the tier `$tier` holds. +macro_rules! with_encode_tier { + ($tier:expr, $k:ident => $body:expr) => {{ + use $crate::encoding::fastpath as tiers; + match $tier { + tiers::FastpathKernel::Scalar => { + type $k = tiers::ScalarTier; + $body + } + #[cfg(all( + target_arch = "aarch64", + target_endian = "little", + feature = "kernel-neon" + ))] + tiers::FastpathKernel::Neon => { + type $k = tiers::NeonTier; + $body + } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" + ))] + tiers::FastpathKernel::Sse2 => { + type $k = tiers::Sse2Tier; + $body + } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" + ))] + tiers::FastpathKernel::Sse42 => { + type $k = tiers::Sse42Tier; + $body + } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] + tiers::FastpathKernel::Avx2Bmi2 => { + type $k = tiers::Avx2Bmi2Tier; + $body + } + #[cfg(all( + target_arch = "wasm32", + target_feature = "simd128", + feature = "kernel-simd128" + ))] + tiers::FastpathKernel::Simd128 => { + type $k = tiers::Simd128Tier; + $body + } + } + }}; +} +pub(crate) use with_encode_tier; + +/// A `match` on `$k`'s [`EncodeTier::TIER`] that instantiates only the arm of +/// that tier. /// -/// # Safety -/// `lhs` / `rhs` must each point to at least `max` initialized bytes. -#[inline(always)] -pub(crate) unsafe fn dispatch_common_prefix_len_ptr_with_kernel( - kernel: FastpathKernel, - lhs: *const u8, - rhs: *const u8, - max: usize, -) -> usize { - match kernel { - FastpathKernel::Scalar => unsafe { scalar::common_prefix_len_ptr(lhs, rhs, max) }, - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - FastpathKernel::Neon => unsafe { neon::common_prefix_len_ptr(lhs, rhs, max) }, - // Both x86 tiers share this kernel: it is SSE2 throughout, and only - // the optimal parser's price set differs between them. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" - ))] - FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { - sse2::common_prefix_len_ptr(lhs, rhs, max) - }, - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" - ))] - FastpathKernel::Avx2Bmi2 => unsafe { avx2_bmi2::common_prefix_len_ptr(lhs, rhs, max) }, - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - FastpathKernel::Simd128 => unsafe { simd128::common_prefix_len_ptr(lhs, rhs, max) }, +/// A plain `match` on the constant folds in an optimised build, but an +/// unoptimised one still instantiates every arm's callee for every tier, so +/// each tier's monomorph would carry the kernels of all the others. Each arm +/// here is an `if` on an inline `const`, whose untaken branch the compiler +/// drops before code generation. The arms are checked for exhaustiveness as the +/// `match` they stand for. +macro_rules! tier_match { + ($k:ty { $( $(#[$attr:meta])* $pat:pat => $arm:expr ),+ $(,)? }) => {{ + const { + match <$k as $crate::encoding::fastpath::EncodeTier>::TIER { + $( $(#[$attr])* $pat => (), )+ + } + }; + 'tier: { + $( + $(#[$attr])* + if const { + matches!(<$k as $crate::encoding::fastpath::EncodeTier>::TIER, $pat) + } { + break 'tier ($arm); + } + )+ + unreachable!() + } + }}; +} +pub(crate) use tier_match; + +/// [`with_encode_tier!`] on this CPU's tier, the one the encoder picks: unit +/// tests drive a backend directly through the kernels it would ship with. +#[cfg(test)] +macro_rules! with_host_tier { + ($k:ident => $body:expr) => { + $crate::encoding::fastpath::with_encode_tier!( + $crate::encoding::fastpath::select_kernel(), + $k => $body + ) + }; +} +#[cfg(test)] +pub(crate) use with_host_tier; + +/// Every tier this CPU may execute, scalar first: each SIMD arm assumes its +/// target feature is present, so a test forcing a tier takes it from here. A +/// test comparing tiers checks the scalar fallback against every SIMD kernel +/// the machine has, including the ones the detection leaves cold. +#[cfg(test)] +pub(crate) fn runnable_tiers() -> alloc::vec::Vec { + #[allow(unused_mut)] + let mut tiers = alloc::vec![FastpathKernel::Scalar]; + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" + ))] + { + if std::is_x86_feature_detected!("sse2") { + tiers.push(FastpathKernel::Sse2); + } + if std::is_x86_feature_detected!("sse4.2") { + tiers.push(FastpathKernel::Sse42); + } + } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] + if std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("bmi2") { + tiers.push(FastpathKernel::Avx2Bmi2); + } + #[cfg(all( + target_arch = "aarch64", + target_endian = "little", + feature = "kernel-neon" + ))] + tiers.push(FastpathKernel::Neon); + #[cfg(all( + target_arch = "wasm32", + target_feature = "simd128", + feature = "kernel-simd128" + ))] + tiers.push(FastpathKernel::Simd128); + tiers +} + +/// `run` under every [`runnable_tiers`] tier, asserting they all agree, and +/// the answer they agree on. +#[cfg(test)] +pub(crate) fn same_under_every_tier( + run: impl Fn(FastpathKernel) -> R, +) -> R { + let tiers = runnable_tiers(); + let answer = run(tiers[0]); + for &tier in &tiers[1..] { + assert_eq!( + run(tier), + answer, + "tier {tier:?} disagrees with {:?}", + tiers[0] + ); } + answer +} + +/// A test's sequence closure as the one sink type tests hand a backend, so +/// each search is compiled once per tier for all of them rather than once per +/// closure and tier. +#[cfg(test)] +pub(crate) fn dyn_sink( + sink: &mut impl FnMut(crate::encoding::Sequence), +) -> &mut dyn FnMut(crate::encoding::Sequence) { + sink } #[cfg(test)] diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index bb4c756c2..c65159e8f 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -9,8 +9,8 @@ use core::hash::Hasher; use super::{ CompressionLevel, LiteralCompressionMode, Matcher, block_header::BlockHeader, - frame_header::FrameHeader, levels::*, match_generator::MatchGeneratorDriver, - workspace::IngestBuffer, + fastpath::EncodeTier, frame_header::FrameHeader, levels::*, + match_generator::MatchGeneratorDriver, workspace::IngestBuffer, }; use crate::common::MAX_BLOCK_SIZE; use crate::fse::fse_encoder::{FSETable, default_ll_table, default_ml_table, default_of_table}; @@ -1207,7 +1207,7 @@ pub(crate) fn optimal_block_size_with( /// again would commit it into the window a second time. Otherwise it is /// primed, or restored from the matcher's copy-mode snapshot when the source /// is past the strategy's attach cutoff. -pub(crate) fn load_frame_dictionary( +pub(crate) fn load_frame_dictionary( state: &mut CompressState, level: CompressionLevel, dict: &EncoderDictionary, @@ -1265,9 +1265,14 @@ pub(crate) fn load_frame_dictionary( ); let restored = prefer_copy_snapshot && state.matcher.restore_primed_dictionary(level); if !restored { - state - .matcher - .prime_with_dictionary(dict.inner.dict_content.as_slice(), dict.inner.offset_hist); + let content = dict.inner.dict_content.as_slice(); + // The built-in matcher indexes the dictionary under the frame's tier. + match state.matcher.builtin_driver() { + Some(driver) => driver.prime_with_dictionary_with::(content, dict.inner.offset_hist), + None => state + .matcher + .prime_with_dictionary(content, dict.inner.offset_hist), + } if prefer_copy_snapshot { state.matcher.capture_primed_dictionary(level); } @@ -2010,8 +2015,7 @@ impl FrameCompressor { return attached; } // The borrowed (no-copy, in-place over-window) scan exists for the - // Simple (Fast), Dfast, and Row backends, and for the HashChain - // backend's lazy CHAIN parser; BT/optimal (BinaryTree search) stay on + // Simple (Fast), Dfast, and Row backends; the optimal parsers stay on // the owned path. Every borrowed scan applies the per-position // `window_low = abs_ip - advertised_window` offset cap so over-window // inputs are matched in place (no input->history copy), matching C's @@ -2026,7 +2030,12 @@ impl FrameCompressor { /// an in-place `&[u8]` cursor. Returns `total_uncompressed`; the caller /// emits the frame header (before this call, when the content size is /// known) or the drain tail. - fn run_one_frame(&mut self, input: &[u8], prep: &FramePrep, out: &mut Vec) -> u64 { + fn run_one_frame( + &mut self, + input: &[u8], + prep: &FramePrep, + out: &mut Vec, + ) -> u64 { if self.borrowed_eligible(input.len(), prep) { // A reused Dfast matcher's tables hold earlier frames, which its // borrowed kernel (numbering this input from zero) would read as @@ -2041,10 +2050,10 @@ impl FrameCompressor { if !self.state.matcher.borrowed_frame_is_independent() { self.state.matcher.forget_earlier_frames(); } - self.run_borrowed_block_loop(input, out) + self.run_borrowed_block_loop::(input, out) } else { let mut cursor: &[u8] = input; - self.run_owned_block_loop(&mut cursor, prep.initial_size_hint, true, out) + self.run_owned_block_loop::<_, K>(&mut cursor, prep.initial_size_hint, true, out) } } @@ -2092,11 +2101,23 @@ impl FrameCompressor { /// Panics on encoder error, matching [`Self::compress`] and /// [`compress_slice_to_vec`](crate::encoding::compress_slice_to_vec). pub fn compress_independent_frame_into(&mut self, input: &[u8], out: &mut Vec) { + // The frame's one choice of kernel tier: its setup, every block's cut, + // search and entropy stage run compiled for it. + crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => self.compress_independent_frame_with::(input, out) + ) + } + + /// [`Self::compress_independent_frame_into`] compiled for the kernel tier + /// `K`. + fn compress_independent_frame_with(&mut self, input: &[u8], out: &mut Vec) { // Size the next frame from the actual payload, not a stale hint a // previous call may have left behind (a wrong hint would change the // resolved window/header and could flip borrowed eligibility). self.source_size_hint = Some(input.len() as u64); - let prep = self.prepare_frame(crate::encoding::workspace::IngestPlan::Slice(input.len())); + let prep = + self.prepare_frame::(crate::encoding::workspace::IngestPlan::Slice(input.len())); // Content size is known up front (one-shot), so write the frame // header FIRST and emit blocks STRAIGHT into `out` — no separate // `all_blocks` accumulator and no header+blocks copy (which was the @@ -2120,7 +2141,7 @@ impl FrameCompressor { out.reserve(18 + first_block_bound + checksum_len); self.append_frame_header(total_uncompressed, &prep, out); let header_len = out.len(); - let _ = self.run_one_frame(input, &prep, out); + let _ = self.run_one_frame::(input, &prep, out); #[cfg(feature = "hash")] if self.content_checksum { out.extend_from_slice(&(self.hasher.finish() as u32).to_le_bytes()); @@ -2161,7 +2182,7 @@ impl FrameCompressor { /// total_uncompressed)`. Caller guarantees Fast backend + no /// dictionary; over-window inputs are fine (matches are bounded by /// `window_low` exactly as the owned evicting path). - fn run_borrowed_block_loop(&mut self, input: &[u8], out: &mut Vec) -> u64 { + fn run_borrowed_block_loop(&mut self, input: &[u8], out: &mut Vec) -> u64 { // Blocks are appended to `out` starting here. `out` may already hold // the frame header (the one-shot compress-into-Vec path writes it // first, since the content size is known up front, and the loop @@ -2235,16 +2256,19 @@ impl FrameCompressor { let savings = start as i64 - (out.len() - blocks_start) as i64; // Borrowed path only: warm the pre-split window before the // cache-cold strided fingerprint read. Gated to exactly the - // conditions under which `optimal_block_size` reads `block` - // (a pre-split level, a full 128 KiB block remaining, the - // block-size cap admits a full block, and `savings >= 3` so the - // splitter actually runs) — so non-pre-split levels, the first - // block, and the trailing partial block pay nothing. See - // `warm_presplit_window`. + // conditions under which `optimal_block_size` reads the whole + // window (a sampling pre-split level, a full 128 KiB block + // remaining, the block-size cap admits a full block, and + // `savings >= 3` so the splitter actually runs) — so other + // levels, the first block, and the trailing partial block pay + // nothing. Level 0 reads three 512-byte segments + // (`ZSTD_splitBlock_fromBorders`, `zstd_preSplit.c:198-226`), not + // the window, so warming 128 KiB for it was 2048 loads a block + // for 1.5 KiB read. See `warm_presplit_window`. if savings >= 3 && input.len() - start >= MAX_BLOCK_SIZE as usize && block_capacity >= MAX_BLOCK_SIZE as usize - && pre_split.is_some() + && pre_split.is_some_and(|split_level| split_level != 0) { warm_presplit_window(&input[start..start + MAX_BLOCK_SIZE as usize]); } @@ -2262,7 +2286,7 @@ impl FrameCompressor { if self.content_checksum { self.hasher.write(block); } - crate::encoding::levels::compress_block_encoded_borrowed( + crate::encoding::levels::compress_block_encoded_borrowed::( &mut *guard.0, self.compression_level, last_block, @@ -2693,10 +2717,20 @@ impl FrameCompressor self.compress_with::() + ) + } + + /// [`Self::compress`] compiled for the kernel tier `K`. + fn compress_with(&mut self) { // `prepare_frame` / `finish_frame` are shared with the borrowed // one-shot path, so both run the same reset / dict-prime / // entropy-seed setup and frame tail. - let prep = self.prepare_frame(crate::encoding::workspace::IngestPlan::Stream); + let prep = self.prepare_frame::(crate::encoding::workspace::IngestPlan::Stream); // Take the reader out so `run_owned_block_loop` can borrow it // mutably alongside `&mut self` (the rest of the loop touches // `self.state` / `self.hasher`, disjoint from the reader). Restored @@ -2722,7 +2756,7 @@ impl FrameCompressor( &mut block_source, prep.initial_size_hint, false, @@ -2746,7 +2780,16 @@ impl FrameCompressor) { - let prep = self.prepare_frame(crate::encoding::workspace::IngestPlan::Stream); + crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => self.compress_known_into_with::(total, out) + ) + } + + /// [`Self::compress_known_into`] compiled for the kernel tier `K`. + #[cfg(feature = "dict-builder")] + fn compress_known_into_with(&mut self, total: u64, out: &mut Vec) { + let prep = self.prepare_frame::(crate::encoding::workspace::IngestPlan::Stream); let mut source = self .uncompressed_data .take() @@ -2755,7 +2798,12 @@ impl FrameCompressor( + &mut block_source, + prep.initial_size_hint, + false, + out, + ); self.uncompressed_data = Some(source); assert_eq!( read, total, @@ -2777,7 +2825,10 @@ impl FrameCompressor FramePrep { + fn prepare_frame( + &mut self, + ingest: crate::encoding::workspace::IngestPlan, + ) -> FramePrep { // The raw-skip's memory of what this frame has already emitted. Frames // are independent, so carrying it over would let one frame's content // hold the skip off for the next; the allocation is kept. @@ -2878,7 +2929,7 @@ impl FrameCompressor( &mut self.state, self.compression_level, dict, @@ -2976,7 +3027,7 @@ impl FrameCompressor( + fn run_owned_block_loop( &mut self, source: &mut S, initial_size_hint: Option, @@ -3061,7 +3112,10 @@ impl FrameCompressor driver.settle_block_cut_with::(available, split_at), + None => self.state.matcher.settle_block_cut(available, split_at), + } } else { split_at }; @@ -3147,7 +3201,7 @@ impl FrameCompressor( &mut self.state, self.compression_level, last_block, diff --git a/zstd/src/encoding/frame_compressor/tests.rs b/zstd/src/encoding/frame_compressor/tests.rs index c464af088..9205c59a8 100644 --- a/zstd/src/encoding/frame_compressor/tests.rs +++ b/zstd/src/encoding/frame_compressor/tests.rs @@ -4800,3 +4800,95 @@ fn estimated_heap_size_bounds_every_prepared_form() { ); } } + +/// The frames one compressor writes for `input` under `tier` through each of +/// its entry points, and through a matcher reaching the built-in driver only +/// by the trait. +fn frames_under_tier( + tier: crate::encoding::fastpath::FastpathKernel, + level: super::CompressionLevel, + input: &[u8], + dict: bool, +) -> Vec> { + use crate::encoding::test_support::{TraitDriver, tier_dictionary}; + let builtin = || { + let mut compressor: FrameCompressor<&[u8], &mut Vec> = FrameCompressor::new(level); + compressor.state.copy_kernel = tier; + if dict { + compressor.set_dictionary(tier_dictionary()).unwrap(); + } + compressor + }; + let mut frames = Vec::new(); + + let mut read = Vec::new(); + let mut compressor = builtin(); + compressor.set_source(input); + compressor.set_drain(&mut read); + compressor.compress(); + drop(compressor); + frames.push(read); + + let mut one_shot = Vec::new(); + builtin().compress_independent_frame_into(input, &mut one_shot); + frames.push(one_shot); + + #[cfg(feature = "dict-builder")] + { + let mut known = Vec::new(); + let mut compressor = builtin(); + compressor.set_source(input); + compressor.compress_known_into(input.len() as u64, &mut known); + frames.push(known); + } + + let mut through_trait = Vec::new(); + let mut compressor = FrameCompressor::new_with_matcher(TraitDriver::new(tier), level); + compressor.state.copy_kernel = tier; + if dict { + compressor.set_dictionary(tier_dictionary()).unwrap(); + } + compressor.set_source(input); + compressor.set_drain(&mut through_trait); + compressor.compress(); + drop(compressor); + frames.push(through_trait); + frames +} + +/// Every tier the CPU can run writes the same frames, with and without a +/// dictionary, at a level of each backend, and every frame decodes to its +/// input. The scalar tier is what a CPU without the extended instructions +/// runs, and the detection never picks it on one that has them, so this is +/// where the fallback is checked end to end against the SIMD kernels. +#[test] +fn every_tier_writes_the_same_frames() { + use crate::encoding::test_support::{TIER_LEVELS, tier_dictionary, tier_fixture}; + let input = tier_fixture(); + let tiers = crate::encoding::fastpath::runnable_tiers(); + for level in TIER_LEVELS { + let level = super::CompressionLevel::Level(level); + for dict in [false, true] { + let reference = frames_under_tier(tiers[0], level, &input, dict); + for frame in &reference { + let mut decoder = FrameDecoder::new(); + if dict { + decoder.add_dict(tier_dictionary()).unwrap(); + } + let mut decoded = Vec::with_capacity(input.len()); + decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); + assert!( + decoded == input, + "{level:?} dict {dict}: a frame does not decode to its input" + ); + } + for &tier in &tiers[1..] { + assert!( + frames_under_tier(tier, level, &input, dict) == reference, + "{level:?} dict {dict}: tier {tier:?} wrote other frames than {:?}", + tiers[0], + ); + } + } + } +} diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs index 08d7e5d21..3c746df8a 100644 --- a/zstd/src/encoding/hc/generator.rs +++ b/zstd/src/encoding/hc/generator.rs @@ -1,61 +1,29 @@ -//! HashChain / binary-tree match generator (`HcMatchGenerator`). +//! Binary-tree match generator (`HcMatchGenerator`) of the optimal parsers. //! -//! The HC chain matcher plus the BT-backed optimal-parse machinery the lazy / -//! btopt / btultra strategies run on: the `HcBackend` discriminator, the -//! `HcMatchGenerator` storage + methods, the `btlazy2` cost helpers, the -//! optimal-plan body macros, and the `build_optimal_plan` driver. Moved -//! verbatim from `match_generator.rs` (no behaviour change); encoding-level -//! paths are absolute (`crate::encoding::…`) for the deeper module. +//! The BT-backed optimal-parse machinery the btopt / btultra / btultra2 +//! strategies run on: the `HcMatchGenerator` storage + methods, the +//! optimal-plan body macros, and the `build_optimal_plan` driver; +//! encoding-level paths are absolute (`crate::encoding::…`) for the deeper +//! module. +#[cfg(test)] use crate::encoding::Sequence; -use crate::encoding::blocks::encode_offset_with_history; use crate::encoding::cost_model::HC_PREDEF_THRESHOLD; -use crate::encoding::hc::{HC_MIN_MATCH_LEN, HcMatch, MAX_HC_SEARCH_DEPTH}; +use crate::encoding::fastpath::EncodeTier; use crate::encoding::levels::config::HcConfig; -use crate::encoding::match_generator::{HC_SEARCH_DEPTH, HC_TARGET_LEN, HcBackend}; +use crate::encoding::match_generator::{HC_SEARCH_DEPTH, HC_TARGET_LEN}; use crate::encoding::match_table::storage::HC3_HASH_LOG; -impl HcBackend { - /// Heap bytes held by the backend. `Hc` is zero-sized; `Bt` boxes a - /// `BtMatcher`, so count the boxed payload plus its own scratch heap. - pub(crate) fn heap_size(&self) -> usize { - match self { - Self::Hc => 0, - Self::Bt(bt) => core::mem::size_of::() + bt.heap_size(), - } - } - - /// Mutable accessor on the BT matcher; panics if the active - /// backend is `Hc`. The HC-or-Bt branches in orchestrator code use - /// `let HcBackend::Bt(bt) = &self.backend` directly for readonly - /// access — this helper exists so macro bodies that already drive - /// a mutable BT update through the optimal parser can write - /// `$self.backend.bt_mut().X` without an outer `match` ladder. - #[inline(always)] - pub(crate) fn bt_mut(&mut self) -> &mut crate::encoding::bt::BtMatcher { - match self { - Self::Bt(bt) => bt, - Self::Hc => unreachable!("BT-only accessor called in HC mode"), - } - } -} - #[derive(Clone)] pub(crate) struct HcMatchGenerator { - /// Shared match-finder storage (window, history, hash / chain / - /// hash3 tables, dictionary-priming flags). Used identically by HC - /// and BT modes; backend-specific table interpretation lives in the - /// matcher methods on this struct. + /// Shared match-finder storage (window, history, hash / tree / + /// hash3 tables, dictionary-priming flags). pub(crate) table: crate::encoding::match_table::storage::MatchTable, - /// HC runtime knobs (lazy_depth, search_depth, target_len). Always - /// present — BT modes still consult `hc.search_depth` for repcode - /// probing and chain candidate enumeration. + /// Search knobs (search_depth, target_len). pub(crate) hc: crate::encoding::hc::HcMatcher, - /// Backend discriminator. [`HcBackend::Hc`] is zero-sized for the - /// lazy / lazy2 path so HC-only generators don't carry the BT - /// optimal-parser scratch buffers. [`HcBackend::Bt`] holds the - /// `BtMatcher` when an optimal mode is configured. - pub(crate) backend: HcBackend, + /// The optimal parser's per-frame state: cost model, scratch arenas and + /// long-distance candidates. Boxed so the generator stays small. + pub(crate) bt: alloc::boxed::Box, /// Compile-time strategy tag mirrored from /// [`MatchGeneratorDriver::strategy_tag`](crate::encoding::match_generator::MatchGeneratorDriver::strategy_tag) /// during `configure()`. @@ -66,8 +34,7 @@ pub(crate) struct HcMatchGenerator { /// const-generic optimal parser (BtOpt vs BtUltra vs BtUltra2). /// Without this field the test path would have to collapse /// `BtOpt` and `BtUltra` onto the same monomorphisation since - /// `table.uses_bt` / `table.is_btultra2` alone can't tell them - /// apart. + /// `table.is_btultra2` alone can't tell them apart. pub(crate) strategy_tag: crate::encoding::strategy::StrategyTag, } @@ -95,10 +62,9 @@ pub(crate) struct HcMatchGenerator { /// re-exported only inside this crate via `pub(crate) use` below. macro_rules! bt_insert_range_body { ($table:expr, $search_depth:expr, $from:ident, $stop:ident, $current_abs_end:ident, $target_abs:ident, $cmf:path) => {{ - // Borrowed-aware live region (owned: `history[history_start..]`; - // borrowed: the in-place input `[0, block_end)`). Reborrow-then-raw-ptr - // so the slice holds NO borrow and coexists with the `&mut $table` - // binary-tree writes below. Owned is byte-identical (same bytes). + // The live region `history[history_start..]`, reborrowed as a raw + // slice so it holds NO borrow and coexists with the `&mut $table` + // binary-tree writes below. let concat: &[u8] = unsafe { let lh = $table.live_history(); core::slice::from_raw_parts(lh.as_ptr(), lh.len()) @@ -344,161 +310,6 @@ macro_rules! bt_insert_range_body { } pub(crate) use bt_insert_range_body; -/// `hash3_candidate` body parameterized over the per-CPU -/// `common_prefix_len_ptr` symbol. The hash3 probe checks one candidate per -/// position when invoked, so the per-call ABI savings compound across the -/// segment. Crate-private (see `bt_insert_range_body!`). -macro_rules! hash3_candidate_body { - ( - $table:expr, - $abs_pos:ident, - $current_abs_end:ident, - $min_match_len:ident, - $cpl:path $(,)? - ) => {{ - if $table.hash3_log == 0 { - return None; - } - let idx = $abs_pos.checked_sub($table.history_abs_start)?; - let concat = $table.live_history(); - if idx + 4 > concat.len() { - return None; - } - let hash3 = $crate::encoding::match_table::storage::MatchTable::hash_position_at( - concat, - idx, - $table.hash3_log, - 3, - ); - let entry = $table - .hash3_table() - .get(hash3) - .copied() - .unwrap_or($crate::encoding::match_table::storage::HC_EMPTY); - let candidate_abs = - $crate::encoding::match_table::storage::MatchTable::stored_abs_position_fast( - entry, - $table.position_base, - $table.index_shift, - )?; - if candidate_abs < $table.history_abs_start || candidate_abs >= $abs_pos { - return None; - } - let offset = $abs_pos - candidate_abs; - if offset >= $crate::encoding::bt::HC3_MAX_OFFSET { - return None; - } - let candidate_idx = candidate_abs - $table.history_abs_start; - let tail_limit = $current_abs_end.saturating_sub($abs_pos); - let base = concat.as_ptr(); - // SAFETY: candidate/idx are within history range; tail_limit - // bounds the scan within `concat`. - let match_len = unsafe { $cpl(base.add(candidate_idx), base.add(idx), tail_limit) }; - (match_len >= $min_match_len).then_some($crate::encoding::opt::types::MatchCandidate { - start: $abs_pos, - offset, - match_len, - }) - }}; -} -pub(crate) use hash3_candidate_body; - -/// `for_each_repcode_candidate_with_reps` body parameterized over the per-CPU -/// `common_prefix_len_ptr` symbol so the per-rep prefix probe inlines under -/// the wrapper's `target_feature` umbrella instead of crossing the ABI -/// boundary through the dispatcher. Three rep probes per encoded position → -/// thousands per segment, so the per-call barrier was non-trivial. -/// -/// The callback `f` runs in the wrapper's umbrella context too, so closures -/// that capture mutable state still work (FnMut). Crate-private -/// (see `bt_insert_range_body!`). -macro_rules! for_each_repcode_candidate_body { - ( - $table:expr, - $abs_pos:ident, - $lit_len:ident, - $reps:ident, - $current_abs_end:ident, - $min_match_len:ident, - $f:ident, - $cpl:path $(,)? - ) => {{ - let rep_offsets: [Option; 3] = if $lit_len == 0 { - [ - Some($reps[1] as usize), - Some($reps[2] as usize), - ($reps[0] > 1).then_some(($reps[0] - 1) as usize), - ] - } else { - [ - Some($reps[0] as usize), - Some($reps[1] as usize), - Some($reps[2] as usize), - ] - }; - let concat = $table.live_history(); - let current_idx = $abs_pos - $table.history_abs_start; - if current_idx + 4 > concat.len() { - return; - } - let tail_limit = $current_abs_end.saturating_sub($abs_pos); - let base = concat.as_ptr(); - let concat_len = concat.len(); - for rep in rep_offsets.into_iter().flatten() { - if rep == 0 || rep > $abs_pos { - continue; - } - let candidate_pos = $abs_pos - rep; - if candidate_pos < $table.history_abs_start { - continue; - } - let candidate_idx = candidate_pos - $table.history_abs_start; - // Upstream zstd `ZSTD_readMINMATCH` gate (zstd_opt.c:657-674): a - // 4-byte (3-byte when min_match_len == 3) equality probe - // before the full prefix scan. Equivalent filtering — a - // mismatch here means `match_len < min_match_len`, which - // the post-scan check rejects anyway — but it skips the - // prefix-kernel call for the common no-match case (rep - // offsets rarely hit on low-redundancy input). - // - // SAFETY: `current_idx + 4 <= concat_len` (early return - // above) and `candidate_idx < current_idx` (rep >= 1), so - // both 4-byte reads stay inside `concat`. - let gate_matches = unsafe { - let cand = base.add(candidate_idx).cast::().read_unaligned(); - let cur = base.add(current_idx).cast::().read_unaligned(); - if $min_match_len == 3 { - // Compare the low-address 3 bytes regardless of - // endianness: byte-shift on LE, mask via to_le. - (cand.to_le() & 0x00FF_FFFF) == (cur.to_le() & 0x00FF_FFFF) - } else { - cand == cur - } - }; - if !gate_matches { - continue; - } - // SAFETY: `candidate_idx ≤ current_idx < concat_len` (since - // candidate_pos ≤ abs_pos and we early-returned on - // `current_idx + 4 > concat_len`). `max` clamps to the shorter - // remaining run so neither pointer overruns `concat`. - let max = (concat_len - candidate_idx) - .min(concat_len - current_idx) - .min(tail_limit); - let match_len = unsafe { $cpl(base.add(candidate_idx), base.add(current_idx), max) }; - if match_len < $min_match_len { - continue; - } - $f(MatchCandidate { - start: $abs_pos, - offset: rep, - match_len, - }); - } - }}; -} -pub(crate) use for_each_repcode_candidate_body; - /// Common prefix of `$a` and `$b`, capped at `$limit`, with the first eight /// bytes settled in place: `$cpl` is called only when they all match. Most /// probed candidates part within one word, and upstream zstd's inlined @@ -1358,10 +1169,12 @@ macro_rules! bt_insert_and_collect_matches_body { pub(crate) use bt_insert_and_collect_matches_body; impl HcMatchGenerator { - /// Heap bytes this generator owns: the shared match table plus the BT - /// backend's optimal-parser / LDM scratch (the HC knobs are inline). + /// Heap bytes this generator owns: the shared match table plus the boxed + /// optimal-parser / LDM state and its scratch (the knobs are inline). pub(crate) fn heap_size(&self) -> usize { - self.table.heap_size() + self.backend.heap_size() + self.table.heap_size() + + core::mem::size_of::() + + self.bt.heap_size() } pub(crate) fn should_run_btultra2_seed_pass( @@ -1376,9 +1189,7 @@ impl HcMatchGenerator { if !S::TWO_PASS_SEED { return false; } - let HcBackend::Bt(bt) = &self.backend else { - return false; - }; + let bt = &self.bt; // The frame's first block, whatever absolute position the frame starts // at (upstream `curr == ms->window.dictLimit`): the window holding // only this block says so. A reused compressor starts each frame at @@ -1401,17 +1212,11 @@ impl HcMatchGenerator { pub(crate) fn new(max_window_size: usize) -> Self { Self { table: crate::encoding::match_table::storage::MatchTable::new(max_window_size), - hc: crate::encoding::hc::HcMatcher::new(2, HC_SEARCH_DEPTH, HC_TARGET_LEN), - // Default to the zero-sized HC backend; `configure()` swaps - // in a `BtMatcher` only when an optimal strategy lands. - backend: HcBackend::Hc, - // Lazy is the per-construct default — every production - // caller calls `configure()` before the first encode and - // overwrites this. Tests that drive `HcMatchGenerator` - // without calling `configure()` end up in the - // `start_matching_lazy` arm of the test dispatcher, which - // matches the previous default behaviour. - strategy_tag: crate::encoding::strategy::StrategyTag::Lazy, + hc: crate::encoding::hc::HcMatcher::new(HC_SEARCH_DEPTH, HC_TARGET_LEN), + bt: alloc::boxed::Box::new(crate::encoding::bt::BtMatcher::new()), + // Every caller runs `configure()` before the first encode, which + // overwrites this. + strategy_tag: crate::encoding::strategy::StrategyTag::BtOpt, } } @@ -1426,11 +1231,14 @@ impl HcMatchGenerator { // `#[cfg(test)] start_matching` dispatcher can route // BtOpt / BtUltra / BtUltra2 to distinct monomorphisations. self.strategy_tag = tag; - let is_btultra2 = tag == StrategyTag::BtUltra2; - let uses_bt = matches!( - tag, - StrategyTag::BtOpt | StrategyTag::BtUltra | StrategyTag::BtUltra2 + debug_assert!( + matches!( + tag, + StrategyTag::BtOpt | StrategyTag::BtUltra | StrategyTag::BtUltra2 + ), + "only the optimal strategies run on this backend, got {tag:?}" ); + let is_btultra2 = tag == StrategyTag::BtUltra2; // btultra and btultra2 both run the mls=3 hash3 short-match probe // (clevels.h minMatch 3). The `is_btultra2` flag below stays // exclusive to btultra2 because it tweaks the BT rebase boundary, @@ -1444,60 +1252,38 @@ impl HcMatchGenerator { let resize = self.table.hash_log != config.hash_log || self.table.chain_log != config.chain_log || self.table.hash3_log != next_hash3_log; - // Capture the layout flip BEFORE `uses_bt` is overwritten below — it - // feeds the dms invalidation (the dms is keyed by layout too). - let uses_bt_changed = self.table.uses_bt != uses_bt; self.table.hash_log = config.hash_log; self.table.chain_log = config.chain_log; self.table.hash3_log = next_hash3_log; - self.hc.search_depth = if uses_bt { - config.search_depth - } else { - config.search_depth.min(MAX_HC_SEARCH_DEPTH) - }; + self.hc.search_depth = config.search_depth; self.hc.target_len = config.target_len; - // Mirror strategy-derived flags + HC search depth onto MatchTable - // so the BT walker and rebase machinery can read them directly - // without dispatching back through HcMatchGenerator. + // Mirror strategy-derived flags + search depth onto MatchTable so the + // BT walker and rebase machinery can read them directly without + // dispatching back through HcMatchGenerator. self.table.search_depth = self.hc.search_depth; self.table.is_btultra2 = is_btultra2; - self.table.uses_bt = uses_bt; // BT finder hash width, upstream zstd `mls = BOUNDED(4, cParams.minMatch, 6)`, // carried explicitly in the level config so a `target_length` override - // cannot silently flip the finder between 5- and 4-byte hashing. Only - // the BT body reads it; HC/lazy levels leave it at 4. clevels.h - // (srcSize > 256 KiB tier): btlazy2 L13-15 + btopt L16 are minMatch=5, - // btopt L17 is minMatch=4, btultra/btultra2 are minMatch=3 (4-byte main - // hash + the hash3 short-match probe). + // cannot silently flip the finder between 5- and 4-byte hashing. + // clevels.h (srcSize > 256 KiB tier): btopt L16 is minMatch=5, btopt + // L17 is minMatch=4, btultra/btultra2 are minMatch=3 (4-byte main hash + // + the hash3 short-match probe). // The cached dms is keyed by the full (region, layout, mls, hash_log) // shape that `build_dms!` validates on the normal prime path, but the // reborrow fast path in `MatchTable::reset` reuses it on `dms.is_primed()` - // ALONE. A reused-compressor level switch can change the search mls (e.g. - // btlazy2 -> lazy), the table geometry (hash_log / chain_log / hash3, - // captured in `resize`), OR the HC<->BT layout (`uses_bt_changed`) - // independently of each other, and any of them leaves the dms hashed for - // a different shape. Invalidate on ANY so the next dict frame re-primes at - // the new shape (configure runs before reset) instead of probing a - // mismatched dms and silently degrading match quality. Over-invalidation - // only costs a re-prime, which a real shape change needs anyway. + // ALONE. A reused-compressor level switch can change the search mls or + // the table geometry (hash_log / chain_log / hash3, captured in + // `resize`) independently of each other, and either leaves the dms + // hashed for a different shape. Invalidate on ANY so the next dict frame + // re-primes at the new shape (configure runs before reset) instead of + // probing a mismatched dms and silently degrading match quality. + // Over-invalidation only costs a re-prime, which a real shape change + // needs anyway. let mls_changed = self.table.search_mls != config.search_mls; - if resize || mls_changed || uses_bt_changed { + if resize || mls_changed { self.table.dms.invalidate(); } self.table.search_mls = config.search_mls; - // Stage D: promote the backend discriminator. HC modes drop the - // BT scratch buffers entirely; switching back into a BT mode - // allocates a fresh `BtMatcher` on demand. - match (&self.backend, self.table.uses_bt) { - (HcBackend::Hc, true) => { - self.backend = - HcBackend::Bt(alloc::boxed::Box::new(crate::encoding::bt::BtMatcher::new())); - } - (HcBackend::Bt(_), false) => { - self.backend = HcBackend::Hc; - } - _ => {} - } if resize && !self.table.tables.is_empty() { // Drop the tables so they are laid out (or allocated) again at the // new widths. The seams go with the buffer. @@ -1514,303 +1300,59 @@ impl HcMatchGenerator { ml: Option<&crate::fse::fse_encoder::FSETable>, of: Option<&crate::fse::fse_encoder::FSETable>, ) { - if let HcBackend::Bt(bt) = &mut self.backend { - bt.opt_state.seed_dictionary_entropy(huff, ll, ml, of); - } + self.bt.opt_state.seed_dictionary_entropy(huff, ll, ml, of); } pub(crate) fn reset(&mut self) { self.table.reset(); - if let HcBackend::Bt(bt) = &mut self.backend { - bt.reset(); - } + self.bt.reset(); } - /// Backfill positions from the tail of the previous slice that couldn't be - /// hashed at the time (insert_position needs 4 bytes of lookahead). - pub(crate) fn skip_matching(&mut self, incompressible_hint: Option) { - self.table.skip_matching(incompressible_hint); + /// Index the last committed block without searching it. + pub(crate) fn skip_matching(&mut self, incompressible_hint: Option) { + self.table.skip_matching::(incompressible_hint); } /// Runtime-dispatched entry kept only for in-crate tests. Production - /// callers reach the inner loops through - /// [`Self::start_matching_strategy`] / [`MatchGeneratorDriver::compress_block`] - /// which pick the lazy / optimal arm from `S::USE_BT` at - /// monomorphisation time. + /// callers reach the optimal parser through + /// [`MatchGeneratorDriver::compress_block`], monomorphised per strategy. + #[cfg(test)] + pub(crate) fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { + crate::encoding::fastpath::with_encode_tier!( + crate::encoding::fastpath::select_kernel(), + K => self.start_matching_with::(handle_sequence) + ) + } + + /// [`Self::start_matching`] compiled for the tier `K`. #[cfg(test)] - pub(crate) fn start_matching(&mut self, mut handle_sequence: impl FnMut(Sequence)) { + pub(crate) fn start_matching_with( + &mut self, + mut handle_sequence: impl FnMut(Sequence), + ) { use crate::encoding::strategy::{self, StrategyTag}; // Dispatch on the mirrored `strategy_tag` so each test runs // under the same monomorphisation production would pick. // `BtOpt` / `BtUltra` / `BtUltra2` remain distinct here even - // though `table.uses_bt` / `is_btultra2` alone can't separate - // BtOpt from BtUltra. + // though `is_btultra2` alone can't separate BtOpt from BtUltra. match self.strategy_tag { - StrategyTag::Fast | StrategyTag::Dfast | StrategyTag::Greedy | StrategyTag::Lazy => { - self.start_matching_lazy(&mut handle_sequence) + StrategyTag::Fast + | StrategyTag::Dfast + | StrategyTag::Greedy + | StrategyTag::Lazy + | StrategyTag::Btlazy2 => { + unreachable!("only the optimal strategies run on this backend") } - StrategyTag::Btlazy2 => unreachable!("btlazy2 runs on the lazy (Row) backend"), StrategyTag::BtOpt => { - self.start_matching_optimal::(&mut handle_sequence) + self.start_matching_optimal::(&mut handle_sequence) } StrategyTag::BtUltra => { - self.start_matching_optimal::(&mut handle_sequence) + self.start_matching_optimal::(&mut handle_sequence) } StrategyTag::BtUltra2 => { - self.start_matching_optimal::(&mut handle_sequence) - } - } - } - - /// Strategy-aware entry point used by - /// [`MatchGeneratorDriver::compress_block`](crate::encoding::match_generator::MatchGeneratorDriver::compress_block). - /// Branches on `S::USE_BT` — a compile-time `const` — so each - /// monomorphisation keeps exactly one arm: `Lazy` / - /// `Fast` / `Dfast` / `Greedy` see only `start_matching_lazy`, - /// `BtOpt` / `BtUltra` / `BtUltra2` see only - /// `start_matching_optimal`. The inherent test-only - /// `HcMatchGenerator::start_matching` reaches the same arms by - /// runtime-matching on `self.strategy_tag` (the parse-mode field - /// has been removed); production never invokes that path. - pub(crate) fn start_matching_strategy( - &mut self, - handle_sequence: &mut impl FnMut(Sequence), - ) { - debug_assert_eq!( - self.table.uses_bt, - S::USE_BT, - "Strategy::USE_BT disagrees with runtime table.uses_bt at HC dispatch" - ); - if S::USE_BT { - self.start_matching_optimal::(handle_sequence) - } else { - self.start_matching_lazy(handle_sequence) - } - } - - /// Dispatcher: pick the dict-aware monomorph when a separate dms is primed - /// (attach-mode dictionary), else the no-dict monomorph. Mirrors upstream's - /// compile-time `dictMode` split — the `DICT = false` body carries no dms - /// code at all, so the no-dict hot path is unaffected by the dict search. - pub(crate) fn start_matching_lazy(&mut self, handle_sequence: impl FnMut(Sequence)) { - if self.table.dms.is_primed() { - self.start_matching_lazy_impl::(handle_sequence); - } else { - self.start_matching_lazy_impl::(handle_sequence); - } - } - - pub(crate) fn start_matching_lazy_impl( - &mut self, - mut handle_sequence: impl FnMut(Sequence), - ) { - self.table.ensure_tables(); - - // `current_block_range()` is borrowed-aware: owned → last committed - // chunk; borrowed → the staged in-place block range. - let (current_abs_start, current_len) = self.table.current_block_range(); - if current_len == 0 { - return; - } - // Full live history (dict + committed blocks + current block), hoisted - // ONCE for the whole position scan and threaded into every - // `find_best_match` / `pick_lazy_match` call. `live_history()` is - // loop-invariant here (the scan mutates the hash/chain tables + - // `offset_hist` but never the history bytes or length), so re-fetching - // it per find — inside `hash_chain_candidate` + the rep probe, plus - // again for each lazy lookahead at pos+1 / pos+2 — was pure - // per-position overhead. Detached as a raw slice so the loop's - // `&mut self.table` inserts coexist with this `&[u8]`; the scan never - // reallocates the history. - let concat: &[u8] = { - let lh = self.table.live_history(); - unsafe { core::slice::from_raw_parts(lh.as_ptr(), lh.len()) } - }; - // Dict-match-state primed flag, hoisted ONCE for the scan: it is - // block-invariant (the dict is primed before the block) and lives on the - // cold `dms` cacheline, so the per-find `dms.is_primed()` load was a - // measurable hot-path cost (~8% of `hash_chain_candidate` on the - // dict-over-random fixture). The `DICT = false` monomorph ignores it. - let dms_primed = self.table.dms.is_primed(); - - let current_abs_end = current_abs_start + current_len; - self.table - .backfill_boundary_positions(current_abs_start, current_abs_end); - - let mut pos = 0usize; - let mut literals_start = 0usize; - // Lookahead carry (upstream's lazy depth loop searches each position - // ONCE): when the lookahead defers to `abs_pos + 1`, its already-computed - // match is carried here and reused next iteration instead of re-searched. - // `None` means search this position fresh. - let mut carried: Option = None; - // Set when the previous position deferred WITHOUT a carry (depth-2: - // `abs_pos + 2` won but `abs_pos + 1` was cold). If this position then - // misses, the no-match skip below must advance by exactly ONE so it - // cannot hop over the already-proven `abs_pos + 2` winner. - let mut deferred_without_carry = false; - while pos + HC_MIN_MATCH_LEN <= current_len { - let abs_pos = current_abs_start + pos; - let lit_len = pos - literals_start; - let forced_single_step = core::mem::take(&mut deferred_without_carry); - - // Reuse the carried lookahead match (searched WITH the previous - // position already inserted, so its offset-1 candidate is visible) - // or search this position fresh. `find_best_match` returns the - // forward `(offset, length)` in registers (`HcMatch`, 16 bytes). - let best = match carried.take() { - Some(m) => m, - None => self.hc.find_best_match::( - concat, - dms_primed, - &self.table, - abs_pos, - lit_len, - ), - }; - if best.is_match() { - // Insert `abs_pos` BEFORE the lookahead so the `abs_pos + 1` - // probe sees it at offset 1 — upstream inserts the searched - // position during its own search, before the depth loop probes - // the next one. - self.table.insert_position(abs_pos); - if let Some(carry) = self.hc.lazy_decide_carry::( - concat, - dms_primed, - &self.table, - abs_pos, - lit_len, - best, - ) { - // DEFER: the lazy lookahead found a better match one (or two) - // bytes ahead. Advance exactly ONE byte. Reuse the lookahead - // match when it is real (so it is not re-searched — upstream - // searches each position once); the `NONE` sentinel is the - // depth-2 defer-without-carry case (`abs_pos + 2` won but - // `abs_pos + 1` had no match), so search the next fresh and - // pin the following advance to one byte (below) so the skip - // heuristic cannot hop over that `abs_pos + 2` winner. - carried = carry.is_match().then_some(carry); - deferred_without_carry = carried.is_none(); - pos += 1; - continue; - } - // COMMIT `best`. Backward-extend over the literal run (upstream - // `zstd_lazy.c` after rep-vs-chain selection) through the shared - // raw-pointer helper the Row / Dfast probes use — one - // back-extension source, no per-step slice bounds checks. - // `abs_pos` is already inserted above, so the forward span fill - // starts at `abs_pos + 1`. - let ext = crate::encoding::match_table::helpers::extend_backwards_shared( - concat, - self.table.history_abs_start, - abs_pos - best.offset, - abs_pos, - best.match_len, - lit_len, - ); - let match_len = ext.match_len; - self.table - .insert_match_span(abs_pos + 1, ext.start + match_len); - // The backward extension stops at the literal run's start. - let start = ext.start - current_abs_start; - debug_assert!(start >= literals_start); - let literal_len = start - literals_start; - handle_sequence(Sequence::Triple { - literal_len, - offset: best.offset, - match_len, - }); - let _ = encode_offset_with_history( - best.offset as u32, - literal_len as u32, - &mut self.table.offset_hist, - ); - pos = start + match_len; - literals_start = pos; - continue; + self.start_matching_optimal::(&mut handle_sequence) } - // No match found. - self.table.insert_position(abs_pos); - // Lazy skipping (upstream zstd `ZSTD_compressBlock_lazy_generic`, - // zstd_lazy.c:1614): advance faster over runs with no match. - // `step = ((ip - anchor) >> kSearchStrength) + 1` with - // kSearchStrength = 8, where `ip - anchor` is the current - // literal-run length. On compressible input the run stays short - // (step == 1, identical to a 1-byte advance); on incompressible - // / dict-over-random input the run grows so the parser skips - // ahead (one search per `step` positions) instead of searching - // every byte. Skipped positions are not inserted, mirroring - // upstream (it inserts only searched positions during a no-match - // run). Ratio follows upstream (not byte-identical). - // A miss one byte after a depth-2 defer-without-carry must advance by - // exactly one so the skip cannot pass the proven `abs_pos + 2` match. - let step = if forced_single_step { - 1 - } else { - ((pos - literals_start) >> 8) + 1 - }; - pos += step; - // No clamp needed before the tail loop: the search bound and the - // hashable bound are both `pos + HC_MIN_MATCH_LEN <= current_len` - // (HC_MIN_MATCH_LEN == 4 == the insert width), so there is no - // non-searchable-but-hashable anchor to miss. Positions the skip - // jumps over inside the searchable region are intentionally not - // inserted — same as upstream zstd, which advances past them via - // the identical `ip += step` and never hashes them either. - } - - // Insert remaining hashable positions in the tail (the matching loop - // stops at HC_MIN_MATCH_LEN but insert_position only needs 4 bytes). - while pos + 4 <= current_len { - self.table.insert_position(current_abs_start + pos); - pos += 1; } - - if literals_start < current_len { - handle_sequence(Sequence::Literals { - len: current_len - literals_start, - }); - } - } - - /// Register the borrowed input window for the no-copy one-shot path. - /// # Safety - /// `buffer` must outlive the borrowed scans (see `MatchTable`). - pub(crate) unsafe fn set_borrowed_window(&mut self, buffer: &[u8]) { - // SAFETY: forwarded liveness contract. - unsafe { self.table.set_borrowed_window(buffer) }; - } - - pub(crate) fn clear_borrowed_window(&mut self) { - self.table.clear_borrowed_window(); - } - - /// Borrowed (no-copy) equivalent of [`Self::start_matching_lazy`]: stage - /// the in-place block range, then run the same lazy chain parse. The - /// parse reads its range via `current_block_range()` and its bytes via - /// `get_last_space()` / `live_history()`, all borrowed-aware, so the block - /// is scanned in place with the per-position window_low offset cap. - pub(crate) fn start_matching_lazy_borrowed( - &mut self, - block_start: usize, - block_end: usize, - handle_sequence: impl FnMut(Sequence), - ) { - self.table.stage_borrowed_block(block_start, block_end); - self.start_matching_lazy(handle_sequence); - } - - /// Borrowed (no-copy) equivalent of the lazy `skip_matching`: stage the - /// in-place block, then seed positions without an owned-history append. - pub(crate) fn skip_matching_borrowed( - &mut self, - block_start: usize, - block_end: usize, - incompressible_hint: Option, - ) { - self.table.stage_borrowed_block(block_start, block_end); - self.table.skip_matching(incompressible_hint); } } diff --git a/zstd/src/encoding/hc/generator/tests.rs b/zstd/src/encoding/hc/generator/tests.rs index eed86e4a9..1f893af90 100644 --- a/zstd/src/encoding/hc/generator/tests.rs +++ b/zstd/src/encoding/hc/generator/tests.rs @@ -6,6 +6,7 @@ use crate::encoding::CompressionLevel; use crate::encoding::Matcher; #[cfg(test)] use crate::encoding::dfast::DfastMatchGenerator; +use crate::encoding::fastpath::{dyn_sink, with_host_tier}; #[cfg(test)] use crate::encoding::match_generator::MatchGeneratorDriver; use crate::encoding::test_support::BlockReplay; @@ -22,13 +23,13 @@ fn dfast_matches_roundtrip_multi_block_pattern() { matcher.commit_input(&first_block); let mut history = Vec::new(); let mut replay = BlockReplay::new(&first_block); - matcher.start_matching(|seq| replay.apply(&mut history, seq)); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut history, seq)))); assert_eq!(history, first_block); matcher.commit_input(&second_block); let prefix_len = history.len(); let mut replay = BlockReplay::new(&second_block); - matcher.start_matching(|seq| replay.apply(&mut history, seq)); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut history, seq)))); assert_eq!(&history[prefix_len..], second_block.as_slice()); } @@ -82,7 +83,7 @@ fn dfast_accepts_exact_five_byte_match() { let mut saw_five_byte_match = false; let mut saw_longer_match = false; - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if let Sequence::Triple { offset, match_len, .. } = seq @@ -93,7 +94,7 @@ fn dfast_accepts_exact_five_byte_match() { saw_longer_match = true; } } - }); + }))); assert!( saw_five_byte_match, diff --git a/zstd/src/encoding/hc/hc_tests.rs b/zstd/src/encoding/hc/hc_tests.rs deleted file mode 100644 index a03222305..000000000 --- a/zstd/src/encoding/hc/hc_tests.rs +++ /dev/null @@ -1,150 +0,0 @@ -//! Unit coverage for `HcMatcher` paths the encode-level suite -//! doesn't naturally hit: short-suffix early returns on probe -//! helpers, chain-walk self-loop branch, and the lazy-pick -//! "next match is better" decline paths. -use super::*; -use crate::encoding::match_table::storage::MatchTable; - -fn table_with_history(buf: &[u8]) -> MatchTable { - let mut t = MatchTable::new(buf.len().max(8)); - t.history = buf.to_vec().into(); - t.history_start = 0; - t.history_abs_start = 0; - t.window_size = buf.len(); - t.position_base = 0; - t.hash_log = 8; - t.chain_log = 8; - t.hash3_log = 0; - t.ensure_tables(); - // `history` is set directly above; record one live chunk. - t.chunk_lens.push_back(buf.len()); - t -} - -#[test] -fn repcode_candidate_returns_none_when_suffix_too_short() { - let mut t = table_with_history(b"abc"); - t.offset_hist = [1, 2, 3]; - // current_idx + HC_MIN_MATCH_LEN > concat.len() → early no-match. - assert!(!HcMatcher::repcode_candidate(&t, 0, 1).is_match()); -} - -#[test] -fn repcode_candidate_skips_rep_at_history_boundary() { - // rep=5 but abs_pos=4, so candidate_pos would underflow into - // pre-history bytes; the `rep > abs_pos` guard must skip it. - let mut t = table_with_history(b"abcdefgh"); - t.offset_hist = [5, 6, 7]; - // No match possible at abs_pos=4 because every rep aims past - // history start. - let result = HcMatcher::repcode_candidate(&t, 4, 1); - assert!(!result.is_match(), "no rep can land in-range"); -} - -#[test] -fn find_best_match_returns_none_for_short_suffix() { - let hc = HcMatcher::new(2, 4, 32); - let t = table_with_history(b"abc"); - assert!( - !hc.find_best_match::(t.live_history(), false, &t, 0, 1) - .is_match() - ); -} - -/// Forward-length selection (upstream `ZSTD_HcFindBestMatch`): the chain -/// walk keeps the longest FORWARD match (`currentMl > ml`) and applies the -/// single backward extension to THAT winner — a shorter-forward candidate -/// is NOT promoted just because it has more backward (`lit_len`) room. -/// Backward "catch up" happens once, on the forward winner, after the walk; -/// it never changes which candidate wins. -/// -/// Fixture (40 bytes): `"AAAabcdefZMQabcdefIJBAAAabcdefIJKKKKKKKK"`. -/// Probing `abs_pos = 24`: the 4-byte hash at `idx 24` ("abcd") collides -/// with `idx 3` and `idx 12`, so the walk visits `[12, 3]` (LIFO). -/// - `idx 12`: forward 8 (`"abcdefIJ"`), `concat[11] = 'Q'` != -/// `concat[23] = 'A'` so no backward room. Total 8, offset 12. -/// - `idx 3`: forward only 6 (`"abcdef"`), but `concat[0..3] = "AAA"` == -/// `concat[21..24]` so it could backward-extend 3 to a TOTAL of 9 — yet -/// its forward length (6) loses to candidate 12's (8), so it is never -/// selected. The forward winner (12) wins at both `lit_len`s. -#[test] -fn hash_chain_candidate_picks_longest_forward_over_shorter_with_backward_room() { - let mut t = MatchTable::new(64); - t.history = b"AAAabcdefZMQabcdefIJBAAAabcdefIJKKKKKKKK".to_vec().into(); - t.history_start = 0; - t.history_abs_start = 0; - t.window_size = t.history.len(); - t.position_base = 0; - t.hash_log = 8; - t.chain_log = 8; - t.hash3_log = 0; - t.ensure_tables(); - t.chunk_lens.push_back(t.history.len()); - t.insert_positions(0, 24); - - let hc = HcMatcher::new(2, 16, 64); - - // The forward winner (idx 12, forward 8) has no backward room. - let c0 = hc.hash_chain_candidate::(t.live_history(), false, &t, 24); - assert!(c0.is_match(), "forward match must be found"); - assert_eq!(c0.match_len, 8, "longest forward match is 8 (idx 12)"); - assert_eq!( - c0.offset, 12, - "winner is the forward-8 candidate at offset 12" - ); -} - -/// Forward-length ties keep the FIRST-visited candidate (upstream -/// `ZSTD_HcFindBestMatch` uses `currentMl > ml`, strictly-longer, so an -/// equal-length later candidate never displaces the earlier one). The walk -/// is newest-first, so in organic chains "first visited" is the closest -/// (smallest-offset) position anyway; this test hand-wires the chain into a -/// non-monotonic order to pin the tie-break rule itself. -/// -/// Fixture: four 8-byte `"abcdefgh"` chunks at `0 / 9 / 18 / 27`, each -/// followed by a unique terminator (`'A'/'B'/'C'/'D'`) capping cross-chunk -/// forward matches at exactly 8. Probing `abs_pos = 27` with the chain -/// hand-wired to visit pos 9 (offset 18) THEN pos 18 (offset 9): both have -/// forward length 8, so the first-visited (pos 9, offset 18) wins. The -/// self-tightening gate at `ml = 8` also rejects pos 18 (its tail byte is a -/// different chunk terminator), so the equal-length later candidate is -/// skipped before the count — consistent with ties-keep-first. -#[test] -fn hash_chain_candidate_forward_ties_keep_first_visited() { - let mut t = MatchTable::new(64); - t.history = b"abcdefghAabcdefghBabcdefghCabcdefghDZZZZ".to_vec().into(); - assert_eq!(t.history.len(), 40); - t.history_start = 0; - t.history_abs_start = 0; - t.window_size = t.history.len(); - t.position_base = 0; - t.hash_log = 8; - t.chain_log = 8; - t.hash3_log = 0; - t.ensure_tables(); - t.chunk_lens.push_back(t.history.len()); - - let abs_pos = 27usize; - let concat = t.live_history(); - let probe_hash = t.hash_position(&concat[abs_pos..]); - // Hand-wire the chain head + link so the walk surfaces pos 9 first - // (offset 18) then pos 18 (offset 9). `stored = pos + 1`. - t.hash_table_mut()[probe_hash] = 9 + 1; - let chain_mask = (1usize << t.chain_log) - 1; - t.chain_table_mut()[9 & chain_mask] = 18 + 1; - t.chain_table_mut()[18 & chain_mask] = HC_EMPTY; - - let hc = HcMatcher::new(2, 16, 64); - let cand = hc.hash_chain_candidate::(t.live_history(), false, &t, abs_pos); - assert!(cand.is_match(), "walk must still produce a match"); - assert_eq!( - cand.match_len, 8, - "both candidates have an 8-byte forward prefix" - ); - assert_eq!( - cand.offset, 18, - "forward-length ties keep the first-visited candidate (pos 9, \ - offset 18); a value of 9 would mean the equal-length later \ - candidate displaced it (non-upstream gain-based tie-break)" - ); -} diff --git a/zstd/src/encoding/hc/mod.rs b/zstd/src/encoding/hc/mod.rs index 52d7a7b64..086b947cc 100644 --- a/zstd/src/encoding/hc/mod.rs +++ b/zstd/src/encoding/hc/mod.rs @@ -1,17 +1,11 @@ -//! Hash-chain match finder used by `Lazy2`. +//! Binary-tree match generator and optimal parser of the btopt / btultra / +//! btultra2 strategies. //! -//! Hosts the runtime knobs of the lazy parser — the lookahead depth -//! (`lazy_depth`), the chain-walk search budget (`search_depth`), and -//! the "sufficient match length" threshold (`target_len`). Method -//! bodies (chain walk, `insert_position`, `pick_lazy_match`, -//! `start_matching_lazy`) still live on `HcMatchGenerator` and will -//! move onto `impl HcMatcher` in Stage 2b alongside the -//! `&mut MatchTable` thread-through; this stage establishes the -//! ownership boundary so the BT-side extraction in Stage 3 has a -//! clean counterpart type to mirror. +//! [`HcMatcher`] carries the search knobs the tree walk and the optimal +//! parser read; the tables live on the shared +//! [`super::match_table::storage::MatchTable`]. //! -//! Upstream zstd parity reference: `lib/compress/zstd_lazy.c`, -//! `ZSTD_HcFindBestMatch` / `ZSTD_compressBlock_lazy2_generic`. +//! Upstream zstd parity reference: `lib/compress/zstd_opt.c`. #![allow(dead_code)] @@ -20,48 +14,12 @@ pub(crate) mod optimal; pub(crate) mod priceset; use super::cost_model::{HC_FORMAT_MINMATCH, HC_OPT_NUM, HcOptimalCostProfile}; -use super::match_table::helpers::common_prefix_len; -use super::match_table::storage::{HC_EMPTY, MatchTable}; -use super::opt::types::MatchCandidate; -/// Minimum match length emitted by the lazy / lazy2 chain walker. -/// Upstream zstd parity: `MIN_MATCH` in `lib/compress/zstd_lazy.c`. +/// Key width the match tables hash a position on. Upstream zstd parity: +/// `MIN_MATCH` in `lib/compress/zstd_lazy.c`. pub(crate) const HC_MIN_MATCH_LEN: usize = 4; -/// Forward-only match found by the lazy chain walk: the `(offset, length)` pair -/// upstream's `ZSTD_HcFindBestMatch` returns (`size_t matchLength` in `rax` + -/// the offset through `offsetPtr`). 16 bytes, so the find result + the lazy -/// lookahead arg travel in registers (`rax:rdx`) instead of the 24-byte -/// `MatchCandidate` (`start` included) the old find returned by value, which -/// the System V ABI spills to the stack and the lazy loop then copies. The -/// backward extension that produces `start` is deferred to the commit site -/// (upstream does it once in the lazy loop after rep-vs-chain selection), so it -/// never touches the per-position find/compare path. -#[derive(Clone, Copy)] -pub(crate) struct HcMatch { - /// Match offset (`abs_pos - candidate_pos`). Meaningful only when - /// [`HcMatch::is_match`] is true. - pub(crate) offset: usize, - /// Forward match length, or `< HC_MIN_MATCH_LEN` for "no match". - pub(crate) match_len: usize, -} - -impl HcMatch { - /// The "no match found" sentinel (`match_len = 0`). - pub(crate) const NONE: Self = Self { - offset: 0, - match_len: 0, - }; - - /// A real match reaches at least `HC_MIN_MATCH_LEN`; shorter is the seed / - /// sentinel, not an emittable match. - #[inline] - pub(crate) fn is_match(self) -> bool { - self.match_len >= HC_MIN_MATCH_LEN - } -} - -/// Hard cap on chain-walk depth. +/// Hard cap on tree-walk depth. pub(crate) const MAX_HC_SEARCH_DEPTH: usize = 512; /// Capacity the optimal parser reserves for its per-position candidate buffer: @@ -81,883 +39,27 @@ const _: () = { assert!(Btlazy2::MAX_CHAIN_DEPTH <= MAX_HC_SEARCH_DEPTH); }; -/// Hash-chain matcher state used by the `Lazy2` parse mode (and the -/// short-history fast path of the BT cascade's initial pass). -/// -/// Owns only the per-frame *configuration* — the actual chain / hash -/// tables live on the shared -/// [`super::match_table::storage::MatchTable`] that this matcher -/// borrows when it runs. +/// The per-frame search configuration of the tree walk. #[derive(Clone)] pub(crate) struct HcMatcher { - /// Lookahead depth (1 = lazy, 2 = lazy2). Upstream zstd parity: - /// `params->cParams.strategy >= ZSTD_lazy2`. - pub(crate) lazy_depth: u8, - /// Maximum number of chain entries inspected per `find_best_match` - /// call. Upstream zstd parity: `params->cParams.searchLog` (clamped to - /// [`MAX_HC_SEARCH_DEPTH`] for HC mode; BT modes use the unclamped value as their walk - /// budget). + /// Tree compares per position. Upstream zstd parity: + /// `1 << params->cParams.searchLog`. pub(crate) search_depth: usize, - /// "Sufficient" match length — once a candidate reaches this - /// length, the lazy decision short-circuits without checking the - /// next position. Upstream zstd parity: + /// "Sufficient" match length that ends a search. Upstream zstd parity: /// `params->cParams.targetLength`. pub(crate) target_len: usize, } impl HcMatcher { - pub(crate) fn new(lazy_depth: u8, search_depth: usize, target_len: usize) -> Self { + pub(crate) fn new(search_depth: usize, target_len: usize) -> Self { Self { - lazy_depth, search_depth, target_len, } } - /// Upstream zstd "match gain" heuristic: `match_len * 4 - offset_bits`. - /// The lazy lookahead uses this to compare a candidate at the - /// Pick the better of two candidate matches by LENGTH (ties to the smaller - /// offset), the same selector the chain walk and the shared Row/Dfast - /// repcode probe use. Upstream `ZSTD_compressBlock_lazy_generic` compares - /// the repcode against the searched match by length at depth 0 (`ml2 > - /// matchLength`, the repcode keeps ties); a longer or equal-length-closer - /// match wins. `None` arms pass the surviving `Some` through. - pub(crate) fn better_candidate(lhs: HcMatch, rhs: HcMatch) -> HcMatch { - if !lhs.is_match() { - return rhs; - } - if !rhs.is_match() { - return lhs; - } - if rhs.match_len > lhs.match_len - || (rhs.match_len == lhs.match_len && rhs.offset < lhs.offset) - { - rhs - } else { - lhs - } - } - - /// Probe the 3 rep-code offsets (with the upstream zstd `ll0 ↦ rep[0] − 1` - /// fallback) and return the best in-range match. Pure helper — - /// only reads from `MatchTable`, no HcMatcher state needed. - pub(crate) fn repcode_candidate(table: &MatchTable, abs_pos: usize, lit_len: usize) -> HcMatch { - let reps = if lit_len == 0 { - [ - Some(table.offset_hist[1] as usize), - Some(table.offset_hist[2] as usize), - (table.offset_hist[0] > 1).then_some((table.offset_hist[0] - 1) as usize), - ] - } else { - [ - Some(table.offset_hist[0] as usize), - Some(table.offset_hist[1] as usize), - Some(table.offset_hist[2] as usize), - ] - }; - - let concat = table.live_history(); - let current_idx = abs_pos - table.history_abs_start; - if current_idx + HC_MIN_MATCH_LEN > concat.len() { - return HcMatch::NONE; - } - // Raw base pointer for the upstream zstd-style `MEM_read32` 4-byte rep - // gate below (single unaligned load each, no per-rep slice bounds check). - let base_ptr = concat.as_ptr(); - - let mut best = HcMatch::NONE; - for rep in reps.into_iter().flatten() { - if rep == 0 || rep > abs_pos { - continue; - } - let candidate_pos = abs_pos - rep; - if candidate_pos - < table - .history_abs_start - .max(abs_pos.saturating_sub(table.max_window_size)) - { - continue; - } - let candidate_idx = candidate_pos - table.history_abs_start; - // Cheap 4-byte equality gate before the wider SIMD count (upstream - // zstd `MEM_read32` repcode gate in `ZSTD_compressBlock_lazy`). - // `HC_MIN_MATCH_LEN == 4` and `current_idx + 4 <= len` (the early - // return above), so a first-4-byte mismatch can never reach the - // match floor — reject without the vector load+count. Falls through - // to the full count only when the candidate lacks a 4-byte lookahead - // (short tail), so the accepted set is byte-identical to the - // unconditional count. Mirrors the chain-walk gate. - if candidate_idx + 4 <= concat.len() - && unsafe { - MatchTable::read_le_u32_ptr(base_ptr.add(candidate_idx)) - != MatchTable::read_le_u32_ptr(base_ptr.add(current_idx)) - } - { - continue; - } - let match_len = common_prefix_len(&concat[candidate_idx..], &concat[current_idx..]); - if match_len >= HC_MIN_MATCH_LEN { - best = Self::better_candidate( - best, - HcMatch { - offset: rep, - match_len, - }, - ); - } - } - best - } - - /// Best hash-chain match at `abs_pos`, in upstream zstd's - /// `ZSTD_HcFindBestMatch` forward-length model: walk the live chain (and - /// the dms) tracking the longest FORWARD match (`currentMl > ml`, ties - /// keep the closest), gate each candidate on the self-tightening 4-byte - /// tail probe, and extend the single winner backwards over the literal - /// run ONCE at the end. No per-candidate `Option` / gain merge / backward - /// extension — that per-attempt overhead is what the old gain-based walk - /// paid over C. - /// - /// `#[inline(always)]`: folded INTO the out-of-line `find_best_match` so the - /// whole find (rep + chain + dms + selection) is ONE function — upstream's - /// `ZSTD_HcFindBestMatch` shape — reached by a single call per position, not - /// the find->chain->dms call chain that an out-of-line chain walk adds. - #[inline(always)] - pub(crate) fn hash_chain_candidate( - &self, - concat: &[u8], - dms_primed: bool, - table: &MatchTable, - abs_pos: usize, - ) -> HcMatch { - let current_idx = abs_pos - table.history_abs_start; - let history_tail = concat.len(); - if current_idx + HC_MIN_MATCH_LEN > history_tail { - return HcMatch::NONE; - } - - // Chain walk is inlined below — avoids the per-call 4 KiB - // `[usize; MAX_HC_SEARCH_DEPTH]` array that - // [`Self::chain_candidates`] materializes (zero-init on entry, - // memcpy on return). At lazy_depth=2 (L7+) `pick_lazy_match` - // triggers up to three chain walks per committed position, so - // the array form costs ~12 KiB of stack traffic per accepted - // match. Upstream zstd (`zstd_lazy.c` `ZSTD_HcFindBestMatch`) runs a - // single fused loop with no intermediate buffer; mirror that. - // - // `chain_candidates` itself is still alive — the chain-walk - // unit tests drive it directly, and the BT-optimal HC - // candidate collector in `match_generator.rs` consumes it - // through a macro pipeline that inherits the array form. - // Inlining the array out of that BT-optimal callsite is a - // separate, larger refactor; this commit only addresses the - // lazy hot path. - let hash = table.hash_position(&concat[current_idx..]); - let chain_mask = (1usize << table.chain_log) - 1; - let mut cur = table.hash_table()[hash]; - // Cap loop at MAX_HC_SEARCH_DEPTH so a misconfigured - // `search_depth > MAX_HC_SEARCH_DEPTH` (BT modes set it from the - // upstream zstd config, which can exceed our cap) cannot run forever. - let max_chain_steps = self.search_depth.min(MAX_HC_SEARCH_DEPTH); - let mut steps = 0usize; - let history_abs_start = table.history_abs_start; - - // Forward-length match state (upstream `ZSTD_HcFindBestMatch`): `ml` is - // the FORWARD match length, seeded at `MIN_MATCH - 1` so the first - // candidate's self-tightening gate (`match + ml - 3`) degenerates to a - // first-4-byte compare. `best_idx` is the winning candidate's concat - // index. - // - // `found` is tracked ONLY in the `DICT = false` monomorph. `ml >= - // MIN_MATCH` is always exactly equivalent to "a match was found" (ml - // only updates via `current_ml > ml`, whose first hit is >= MIN_MATCH), - // so the `DICT = true` path uses that equivalence and drops the bool — - // the dict monomorph is register-tight (it also runs the dms walk), and - // eliding `found` there frees a stack slot (measured: -5% cycles on the - // dict path). The `DICT = false` monomorph is NOT register-tight, and - // there the equivalent `ml < MIN_MATCH` final check changed codegen - // layout enough to cost ~+4% (same instruction count, worse IPC), so it - // keeps the original bool. `DICT` is const, so each monomorph compiles - // exactly one arm with no runtime branch; in the dict monomorph `found` - // is written/read only in dead `!DICT` arms and is fully eliminated. - let mut ml = HC_MIN_MATCH_LEN - 1; - let mut best_idx = 0usize; - let mut found = false; - // Raw base pointer for the upstream zstd-style `MEM_read32` gate - // (single unaligned load each, no per-candidate slice bounds check). - let base_ptr = concat.as_ptr(); - // Loop-invariant precompute, hoisted out of the chain walk (upstream zstd: - // `lowLimit` + base pointers are set once before the loop). Candidates - // are tracked in history-RELATIVE index space so each step is a single - // `add` + range check + 4-byte gate, mirroring `ZSTD_HcFindBestMatch`'s - // tight body (`matchIndex >= lowLimit`; `NEXT_IN_CHAIN`) instead of an - // `Option`-returning absolute-position decode plus a per-candidate - // `.max()/.saturating_sub()` window-floor recompute. - let floor_idx = current_idx.saturating_sub(table.max_window_size); - // `candidate_idx = position_base + (cur-1) - index_shift - - // history_abs_start`, folded into one wrapping bias so the per-step cost - // is `cur + idx_bias`. Stale / sub-floor chain entries wrap to a huge - // index and are rejected by the `< current_idx` upper bound — the - // accepted set is identical to the `stored_abs_position_fast` decode - // (which returned `None` for the same sub-floor entries). - let idx_bias = table - .position_base - .wrapping_sub(1) - .wrapping_sub(table.index_shift) - .wrapping_sub(history_abs_start); - // Hoist the chain-table base pointer ONCE: through `&MatchTable` the - // optimizer reloads the `Vec` (ptr, len) header — and re-checks bounds — - // on every `table.chain_table[..]`, a measured ~1.7x of upstream's - // per-find load count. `candidate_rel & chain_mask` is always - // `< chain_table.len()` (the table is `1 << chain_log` and - // `chain_mask == len - 1`), so the raw read is in-bounds. - let chain_ptr: *const u32 = table.chain_table().as_ptr(); - while steps < max_chain_steps { - if cur == HC_EMPTY { - break; - } - let candidate_rel = cur.wrapping_sub(1) as usize; - // SAFETY: `candidate_rel & chain_mask < chain_table.len()`. - let next = unsafe { *chain_ptr.add(candidate_rel & chain_mask) }; - steps += 1; - - let candidate_idx = (cur as usize).wrapping_add(idx_bias); - // A wrapped index (`>= current_idx`) means `abs < history_abs_start`: - // a stale entry from a previous frame. The chain is head-insert - // monotonic, so the rest of the tail is older and also stale -- stop. - // This is the early-exit the no-memset floor-advance reset needs; in - // the memset reset the chain has no wrapped entries, so it never - // fires. INVARIANT: the memset reset zeroes every slot to `HC_EMPTY` - // (caught by the `cur == HC_EMPTY` break above) and uses `idx_bias` - // that maps stored positions back to indices `< current_idx`, so no - // surviving entry can wrap -- only the floor-advance path retains - // cross-frame entries that decode past `current_idx`. If a future - // change let a memset-reset chain produce a wrap, this break would - // silently cut the walk short (a miss-rate regression, not a - // correctness bug). Below-floor (`< floor_idx`) entries are NOT a - // break: a chain-slot collision can put an in-window entry after one. - if candidate_idx >= current_idx { - break; - } - if candidate_idx >= floor_idx { - // Upstream zstd's single self-tightening gate (`zstd_lazy.c:714`): - // MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3) - // proves the candidate can possibly reach `ml + 1` before paying - // for the full `common_prefix_len`. `ml >= MIN_MATCH - 1 = 3` so - // `gate_off >= 0`. The iLimit break below keeps - // `current_idx + ml < history_tail` on entry, so - // `current_idx + gate_off + 4 = current_idx + ml + 1 <= history_tail`, - // and `candidate_idx < current_idx` keeps the match-side read in - // range too -- no per-iteration bounds check (matches C's - // branchless gate). - let gate_off = ml - 3; - // SAFETY: both reads are `<= history_tail` per the bound above. - let gate_ok = unsafe { - MatchTable::read_le_u32_ptr(base_ptr.add(candidate_idx + gate_off)) - == MatchTable::read_le_u32_ptr(base_ptr.add(current_idx + gate_off)) - }; - if gate_ok { - let current_ml = - common_prefix_len(&concat[candidate_idx..], &concat[current_idx..]); - // `currentMl > ml`: strictly longer forward match wins; ties - // keep the first (closest, the walk is newest-first). - if current_ml > ml { - ml = current_ml; - best_idx = candidate_idx; - // Dict path reads `ml >= MIN_MATCH`; only the no-dict - // monomorph keeps the `found` flag (see seed comment). - if !DICT { - found = true; - } - // Upstream `if (ip + currentMl == iLimit) break`: reached - // the input end (best possible); also keeps the next gate - // read in bounds. - if current_idx + ml >= history_tail { - break; - } - } - } - } - - // Self-loop (two positions share `candidate_rel & chain_mask`): - // checked here where `next` / `cur` are already live, not carried as - // a bool across the body (a register in this saturated walk). - if next == cur { - break; - } - cur = next; - } - - // Separate dictionary match state (upstream `ZSTD_dictMatchState`). The - // walk is OUT-OF-LINE so the dict-only code never bloats this hot - // function on no-dict frames. It shares `ml` / `steps` (upstream's - // single `nbAttempts` across the live + dms loops). Skip it when the - // live walk already reached iLimit (`ml` maximal, and the dms - // self-tightening gate would read past `history_tail`). - // `dms_primed` is hoisted by the caller (block-invariant — the dict is - // primed before the block scan), replacing a per-find `dms.is_primed()` - // load from the cold `dms` cacheline. - if DICT && dms_primed && current_idx + ml < history_tail { - // dms runs only in the `DICT = true` monomorph, which uses the - // `ml >= MIN_MATCH` match test, so the walk tracks `(ml, best_idx)` - // and never the `found` flag. - let walk = self.dms_chain_walk( - table, - concat, - current_idx, - history_tail, - base_ptr, - max_chain_steps, - steps, - ml, - best_idx, - ); - ml = walk.0; - best_idx = walk.1; - } - - // Dict monomorph: `ml >= MIN_MATCH` is the match test (drops `found`). - // No-dict monomorph: keep the original `found` flag (see seed comment). - let matched = if DICT { ml >= HC_MIN_MATCH_LEN } else { found }; - if !matched { - return HcMatch::NONE; - } - // Forward `(offset, length)` only — the backward extension that produces - // `start` runs once at the commit site (upstream's lazy loop), off the - // per-position find/compare path. `offset = abs_pos - (best_idx + - // history_abs_start) = current_idx - best_idx`. - HcMatch { - offset: current_idx - best_idx, - match_len: ml, - } - } - - /// Out-of-line dms HC4 walk (upstream `ZSTD_HcFindBestMatch` dms loop, - /// `zstd_lazy.c:751-769`). Split from [`Self::hash_chain_candidate`] so the - /// no-dict hot path keeps its small body / register budget. Shares the - /// caller's forward-length state (`ml` / `best_idx`) and `steps` - /// budget -- the live + dms loops are one bounded operation (upstream's - /// single `nbAttempts`). The dict sits at the front of `concat` - /// (`[0, region)`); a dms candidate is a concat index `< current_idx`, so - /// the offset / gate / count logic matches the live walk. Only the - /// `DICT = true` monomorph calls this, and it uses the `ml >= MIN_MATCH` - /// match test, so the walk returns `(ml, best_idx)` and tracks no `found` - /// flag. - /// - /// `#[inline]`: the `DICT = false` monomorph never references this (the - /// `if DICT &&` gate is a compile-time false), so it is dropped from the - /// no-dict path; the `DICT = true` monomorph inlines it into the dict hot - /// path. - #[inline] - #[allow(clippy::too_many_arguments)] - fn dms_chain_walk( - &self, - table: &MatchTable, - concat: &[u8], - current_idx: usize, - history_tail: usize, - base_ptr: *const u8, - max_chain_steps: usize, - mut steps: usize, - mut ml: usize, - mut best_idx: usize, - ) -> (usize, usize) { - let dms = match table.dms.table() { - Some(d) => d, - None => return (ml, best_idx), - }; - // Match upstream's effective dict search depth: the CDict path runs the - // dedicated dict search deeper than the bare level searchLog (measured: - // upstream needs nbAttempts >= 16 to surface the long dict match on the - // per-label-dict fixtures). Floor the shared budget at 16. - // - // NOTE on the per-find step budget: `steps` is SHARED with the live - // walk. The floor below is `max(max_chain_steps, 16)`, so the live and - // dms walks TOGETHER are bounded by it (= 16 at every standard HC level, - // where `max_chain_steps < 16`: L6 `searchLog = 3` -> 8); the dms - // consumes whatever the live walk left. Giving the dms a fresh 16 - // attempts on top of the live walk instead (a per-walk, non-shared - // budget) was measured +3% cycles on the small-10k-random dict path - // with zero ratio change -- the shared floor already surfaces every - // dict match that reaches the output, so sharing it is intentional: the - // floor (not `max_chain_steps`) is the binding dict-search constraint. - let dms_budget = max_chain_steps.max(16); - let dms_hash = MatchTable::hash_position_at(concat, current_idx, dms.hash_log, dms.mls); - let mut dcur = dms.hash_table[dms_hash]; - // Hoist the dms chain-table base pointer (same Vec-header-reload removal - // as the live walk above). - let dms_chain_ptr: *const u32 = dms.chain_table.as_ptr(); - // Reachability floor, hoisted out of the loop. A dict position is - // decoder-reachable iff its offset (`current_idx - dict_idx`) does not - // exceed `max_window_size`, i.e. `dict_idx >= current_idx - - // max_window_size`. Comparing `dict_idx` against this precomputed floor - // drops the per-iteration `current_idx - dict_idx` subtract from the hot - // path (the offset is only needed when a candidate WINS, which is rare on - // the walk) and frees the `max_window_size` register inside the loop. - // `saturating_sub` is the intended business logic, not an overflow guard: - // when `current_idx <= max_window_size` the whole live history fits the - // window, so every dict position is reachable and the floor is 0. Mirrors - // C's `ZSTD_HcFindBestMatch` dms loop, which bounds reachability by chain - // construction rather than re-checking each candidate. - let dict_reachable_floor = current_idx.saturating_sub(table.max_window_size); - while steps < dms_budget { - if dcur == 0 { - break; - } - // Dict position is a concat index in `[0, region)`; the dict is at - // the front so `dict_idx < current_idx` always (and - // `< dms.chain_table.len()`). - let dict_idx = (dcur - 1) as usize; - // SAFETY: `dict_idx` is a stored dict position `< chain_table.len()`. - let dnext = unsafe { *dms_chain_ptr.add(dict_idx) }; - steps += 1; - // Out-of-window dict positions are unreachable to the decoder. - if dict_idx >= dict_reachable_floor { - // Same self-tightening gate as the live walk; the caller's guard - // (`current_idx + ml < history_tail`) + the iLimit break keep - // `current_idx + (ml - 3) + 4 <= history_tail`, and - // `dict_idx < current_idx` keeps the dict-side read in range. - let gate_off = ml - 3; - // SAFETY: both reads are `<= history_tail` per the bound above. - let gate_ok = unsafe { - MatchTable::read_le_u32_ptr(base_ptr.add(dict_idx + gate_off)) - == MatchTable::read_le_u32_ptr(base_ptr.add(current_idx + gate_off)) - }; - if gate_ok { - let current_ml = common_prefix_len(&concat[dict_idx..], &concat[current_idx..]); - if current_ml > ml { - ml = current_ml; - best_idx = dict_idx; - if current_idx + ml >= history_tail { - break; - } - } - } - } - // Chain links are strictly decreasing (head insert); `dnext == dcur` - // (or the `dcur == 0` at the top) ends the walk. - if dnext == dcur { - break; - } - dcur = dnext; - } - (ml, best_idx) - } - - /// Single-rep (offset_1) probe — the lazy parser's inline rep check - /// (upstream `ZSTD_compressBlock_lazy_generic`: only `rep[0]` is tested per - /// position, the 3-rep rotation is the optimal parser's). Caller guarantees - /// `lit_len > 0`, so `rep[0]` is repcode index 0 with no ll0 rotation, and - /// extending it backwards over the literal run is sound. Strips - /// [`Self::repcode_candidate`] to its first rep: one window check, one - /// 4-byte gate, one count — a third of the per-position rep cost the 3-rep - /// probe paid (and which the lazy lookahead paid again at pos+1 / pos+2). - /// - /// `#[inline(always)]`: upstream checks the rep INLINE in the lazy loop; - /// folded into the (out-of-line) `find_best_match` monolith so the rep is - /// part of the single per-position find call, not a nested out-of-line probe. - #[inline(always)] - pub(crate) fn repcode_candidate_offset1( - concat: &[u8], - table: &MatchTable, - abs_pos: usize, - ) -> HcMatch { - let rep = table.offset_hist[0] as usize; - if rep == 0 || rep > abs_pos { - return HcMatch::NONE; - } - let current_idx = abs_pos - table.history_abs_start; - if current_idx + HC_MIN_MATCH_LEN > concat.len() { - return HcMatch::NONE; - } - let candidate_pos = abs_pos - rep; - if candidate_pos - < table - .history_abs_start - .max(abs_pos.saturating_sub(table.max_window_size)) - { - return HcMatch::NONE; - } - let candidate_idx = candidate_pos - table.history_abs_start; - // Cheap 4-byte equality gate before the SIMD count (upstream `MEM_read32` - // repcode gate); identical to the per-rep gate in `repcode_candidate`. - let base_ptr = concat.as_ptr(); - if candidate_idx + 4 <= concat.len() - && unsafe { - MatchTable::read_le_u32_ptr(base_ptr.add(candidate_idx)) - != MatchTable::read_le_u32_ptr(base_ptr.add(current_idx)) - } - { - return HcMatch::NONE; - } - let match_len = common_prefix_len(&concat[candidate_idx..], &concat[current_idx..]); - if match_len >= HC_MIN_MATCH_LEN { - HcMatch { - offset: rep, - match_len, - } - } else { - HcMatch::NONE - } - } - - /// Combine the rep-code and chain-walk candidates and pick the - /// better of the two. Monomorphised over `DICT` (upstream's compile-time - /// `dictMode` template param): the `DICT = false` instance compiles WITHOUT - /// any dms code so the no-dict hot path keeps its tight body, while the - /// `DICT = true` instance dual-probes the separate dictMatchState. The - /// dispatcher in `start_matching_lazy` picks the instance from whether a dms - /// is primed. - /// - /// `#[inline(never)]`: kept out-of-line (upstream's `ZSTD_HcFindBestMatch` - /// is its own function, with the lazy loop thin) so the find's register - /// pressure is contained in its own frame instead of competing with the - /// per-position loop state; the 16-byte `HcMatch` return travels in - /// `rax:rdx`, so the find->loop boundary stays a register pair, not a stack - /// spill. - #[inline(never)] - pub(crate) fn find_best_match( - &self, - concat: &[u8], - dms_primed: bool, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - ) -> HcMatch { - // `concat` is the full live history (dict + committed blocks + current - // block), hoisted ONCE per block by the caller — `live_history()` is - // loop-invariant for the position scan, so fetching it per find (in - // `hash_chain_candidate` + the rep probe) was pure per-position - // overhead. See `start_matching_lazy`'s block-level hoist. - // - // Upstream `ZSTD_compressBlock_lazy_generic` checks only offset_1 - // (`rep[0]`) inline per position; the 3-rep rotation lives in the - // optimal parser, not the lazy loop. For `lit_len > 0` that is exactly - // `rep[0]` (repcode index 0, no ll0 rotation), so probe the single rep. - // The `lit_len == 0` case keeps the full rotated probe for encode - // correctness (the ll0 `rep[0]-1` rule). - let rep = if lit_len == 0 { - Self::repcode_candidate(table, abs_pos, lit_len) - } else { - Self::repcode_candidate_offset1(concat, table, abs_pos) - }; - let hash = self.hash_chain_candidate::(concat, dms_primed, table, abs_pos); - Self::better_candidate(rep, hash) - } - - /// Upstream zstd `lazy` / `lazy2` lookahead: evaluate the match a byte - /// (and optionally two) ahead before committing the current one. - /// Returns `true` if the current match `best` wins (commit it), `false` - /// if the caller should defer to a later position. - /// - /// Takes the already-unwrapped `best` BY VALUE and returns the lazy - /// commit/defer decision: `None` = COMMIT `best`; `Some(carry)` = DEFER and - /// advance one byte. The caller reuses `carry` as the next `best` when it is - /// a real match (`carry.is_match()`), or searches the next position fresh - /// when it is the `NONE` sentinel — the depth-2 defer-WITHOUT-carry case - /// (the `abs_pos + 2` probe wins but `abs_pos + 1` had no match). That case - /// must DEFER, not commit, or lazy2 misses the two-ahead match. - /// - /// The caller inserts `abs_pos` into the hash tables BEFORE calling this, so - /// the `pos + 1` / `pos + 2` lookahead sees `abs_pos` at offset 1 — matching - /// upstream, where the searched position is inserted during its own search, - /// before the depth loop probes the next one. - pub(crate) fn lazy_decide_carry( - &self, - concat: &[u8], - dms_primed: bool, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - best: HcMatch, - ) -> Option { - let history_end = table.history_abs_start + concat.len(); - // HC's finder is out-of-line by design (`#[inline(never)]`), so the - // shared decision macro expands here with no register cost. Map the - // macro's `Option>` to one `Option` level: `None` - // (commit) stays `None`; `Some(Some(carry))` (defer with a reusable - // lookahead) -> `Some(carry)`; `Some(None)` (depth-2 defer with no - // carryable match one byte ahead) -> `Some(NONE)` so the caller still - // DEFERS but searches the next position fresh. - match crate::encoding::lazy_parse::lazy_decide!( - best_len = best.match_len, - best_off = best.offset, - target_len = self.target_len, - lazy_depth = self.lazy_depth, - abs_pos = abs_pos, - lit_len = lit_len, - history_end = history_end, - min_match = HC_MIN_MATCH_LEN, - search = |pos, ll| { - let m = self.find_best_match::(concat, dms_primed, table, pos, ll); - m.is_match().then_some(m) - }, - ) { - ::core::option::Option::None => ::core::option::Option::None, - ::core::option::Option::Some(::core::option::Option::Some(carry)) => Some(carry), - ::core::option::Option::Some(::core::option::Option::None) => Some(HcMatch::NONE), - } - } - - /// Cross-platform dispatcher for the rep-code probe used by the - /// optimal-parser pipeline. Routes to the kernel-specific variant - /// so the per-rep `common_prefix_len_ptr` call inlines under the - /// callee's `target_feature` umbrella. Test / external callers - /// only — the on-encode hot path bypasses this dispatcher via the - /// kernel-specific variants invoked from inside - /// `collect_optimal_candidates_initialized_`. - #[allow(dead_code)] - #[inline(always)] - #[allow(clippy::too_many_arguments)] - pub(crate) fn for_each_repcode_candidate_with_reps( - &self, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - reps: [u32; 3], - current_abs_end: usize, - min_match_len: usize, - f: impl FnMut(MatchCandidate), - ) { - // SAFETY: each branch verifies the target_feature requirement of - // the callee (same shape as the BT walk dispatchers). - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - unsafe { - self.for_each_repcode_candidate_with_reps_neon( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - ) - } - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - use crate::encoding::fastpath::FastpathKernel; - match table.kernel { - #[cfg(feature = "kernel-avx2")] - FastpathKernel::Avx2Bmi2 => unsafe { - self.for_each_repcode_candidate_with_reps_avx2_bmi2( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - ) - }, - #[cfg(feature = "kernel-sse")] - FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { - self.for_each_repcode_candidate_with_reps_sse2( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - ) - }, - FastpathKernel::Scalar => self.for_each_repcode_candidate_with_reps_scalar( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - ), - } - } - #[cfg(not(any( - all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ), - target_arch = "x86", - target_arch = "x86_64" - )))] - { - self.for_each_repcode_candidate_with_reps_scalar( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - ) - } - } - - /// NEON umbrella variant of the rep-code probe. - /// - /// # Safety - /// Caller must be running on an AArch64 target with NEON - /// available (baseline on AArch64). - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - #[target_feature(enable = "neon")] - #[allow(clippy::too_many_arguments)] - pub(crate) unsafe fn for_each_repcode_candidate_with_reps_neon( - &self, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - reps: [u32; 3], - current_abs_end: usize, - min_match_len: usize, - mut f: impl FnMut(MatchCandidate), - ) { - let _ = self; - super::hc::generator::for_each_repcode_candidate_body!( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - crate::encoding::fastpath::neon::common_prefix_len_ptr, - ) - } - - /// SSE2 umbrella variant. - /// - /// # Safety - /// Caller must be running on x86/x86_64 with SSE2 available. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" - ))] - #[target_feature(enable = "sse2")] - #[allow(clippy::too_many_arguments)] - pub(crate) unsafe fn for_each_repcode_candidate_with_reps_sse2( - &self, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - reps: [u32; 3], - current_abs_end: usize, - min_match_len: usize, - mut f: impl FnMut(MatchCandidate), - ) { - let _ = self; - super::hc::generator::for_each_repcode_candidate_body!( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - crate::encoding::fastpath::sse2::common_prefix_len_ptr, - ) - } - - /// AVX2+BMI2 umbrella variant. - /// - /// # Safety - /// Caller must be running on x86/x86_64 with AVX2 + BMI2 available. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" - ))] - #[target_feature(enable = "avx2,bmi2")] - #[allow(clippy::too_many_arguments)] - pub(crate) unsafe fn for_each_repcode_candidate_with_reps_avx2_bmi2( - &self, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - reps: [u32; 3], - current_abs_end: usize, - min_match_len: usize, - mut f: impl FnMut(MatchCandidate), - ) { - let _ = self; - super::hc::generator::for_each_repcode_candidate_body!( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, - ) - } - - /// Scalar fallback, compiled unless the NEON tier covers this target. - #[cfg(not(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - )))] - #[allow(clippy::too_many_arguments)] - pub(crate) fn for_each_repcode_candidate_with_reps_scalar( - &self, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - reps: [u32; 3], - current_abs_end: usize, - min_match_len: usize, - mut f: impl FnMut(MatchCandidate), - ) { - let _ = self; - super::hc::generator::for_each_repcode_candidate_body!( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - crate::encoding::fastpath::scalar::common_prefix_len_ptr, - ) - } - - /// Walk a candidate match backwards over the literal run so the - /// matcher can absorb literal bytes that happen to match the byte - /// preceding the candidate. Upstream zstd parity: equivalent to the back - /// extend inside `ZSTD_HcFindBestMatch` before committing a - /// sequence. - /// - /// Takes `&MatchTable` because the only thing it needs is read - /// access to the contiguous history mirror — kept off `&self` so - /// callers don't have to hand it an HcMatcher reference they - /// don't otherwise use. - pub(crate) fn extend_backwards( - table: &MatchTable, - candidate_pos: usize, - abs_pos: usize, - match_len: usize, - lit_len: usize, - ) -> MatchCandidate { - crate::encoding::match_table::helpers::extend_backwards_shared( - table.live_history(), - table.history_abs_start, - candidate_pos, - abs_pos, - match_len, - lit_len, - ) - } - /// Upstream zstd parity: per-pass clamp of the "good enough — stop probing" - /// threshold that the optimal parser passes to the BT/HC walkers. + /// threshold that the optimal parser passes to the tree walk. /// Reflects upstream zstd `ZSTD_compressBlock_opt_generic` which caps the /// profile's `sufficient_match_len` by the user-configured /// `targetLength` and the `HC_OPT_NUM` ceiling. @@ -968,6 +70,3 @@ impl HcMatcher { .clamp(HC_FORMAT_MINMATCH, HC_OPT_NUM - 1) } } - -#[cfg(test)] -mod hc_tests; diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index 45d0a293b..1f7026cb1 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -16,6 +16,7 @@ use super::generator::HcMatchGenerator; use crate::encoding::Sequence; use crate::encoding::blocks::encode_offset_with_history; +use crate::encoding::fastpath::EncodeTier; use crate::encoding::hc::HC_OPT_CANDIDATE_CAPACITY; use crate::encoding::hc::priceset::{ priceset_avx2, priceset_neon, priceset_scalar, priceset_simd128, priceset_sse2, priceset_sse41, @@ -38,7 +39,7 @@ use crate::encoding::{ }, hc::HC_MIN_MATCH_LEN, levels::config::HcConfig, - match_generator::{HC_OPT_MIN_MATCH_LEN, HC_SEARCH_DEPTH, HC_TARGET_LEN, HcBackend}, + match_generator::{HC_OPT_MIN_MATCH_LEN, HC_SEARCH_DEPTH, HC_TARGET_LEN}, match_table::storage::HC3_HASH_LOG, opt::ldm::{HcOptLdmState, HcRawSeqStore}, opt::types::{ @@ -278,7 +279,7 @@ macro_rules! build_optimal_plan_impl_body { seq_store: HcRawSeqStore { pos: 0, pos_in_sequence: 0, - size: $self.backend.bt_mut().ldm_sequences.len(), + size: $self.bt.ldm_sequences.len(), }, ..HcOptLdmState::default() } @@ -299,13 +300,11 @@ macro_rules! build_optimal_plan_impl_body { // matches that copy the wrong bytes (undecodable frame). if ldm_block_offset > 0 { $self - .backend - .bt_mut() + .bt .ldm_skip_raw_seq_store_bytes(&mut opt_ldm.seq_store, ldm_block_offset); } $self - .backend - .bt_mut() + .bt .ldm_get_next_match_and_update_seq_store(&mut opt_ldm, 0, $current_len); } @@ -313,7 +312,7 @@ macro_rules! build_optimal_plan_impl_body { // at current position before entering the generic forward DP loop. if $current_len >= min_match_len { let seed_ldm = if HAS_LDM { - $self.backend.bt_mut().ldm_process_match_candidate( + $self.bt.ldm_process_match_candidate( &mut opt_ldm, 0, $current_len, @@ -366,15 +365,15 @@ macro_rules! build_optimal_plan_impl_body { HC_OPT_PRICE_STRIDE, ) }; - $self.backend.bt_mut().opt_ll_price_stamp = - $self.backend.bt_mut().opt_ll_price_stamp.wrapping_add(1).max(1); - ll_price_stamp = $self.backend.bt_mut().opt_ll_price_stamp; - $self.backend.bt_mut().opt_lit_price_stamp = - $self.backend.bt_mut().opt_lit_price_stamp.wrapping_add(1).max(1); - lit_price_stamp = $self.backend.bt_mut().opt_lit_price_stamp; - $self.backend.bt_mut().opt_ml_price_stamp = - $self.backend.bt_mut().opt_ml_price_stamp.wrapping_add(1).max(1); - ml_price_stamp = $self.backend.bt_mut().opt_ml_price_stamp; + $self.bt.opt_ll_price_stamp = + $self.bt.opt_ll_price_stamp.wrapping_add(1).max(1); + ll_price_stamp = $self.bt.opt_ll_price_stamp; + $self.bt.opt_lit_price_stamp = + $self.bt.opt_lit_price_stamp.wrapping_add(1).max(1); + lit_price_stamp = $self.bt.opt_lit_price_stamp; + $self.bt.opt_ml_price_stamp = + $self.bt.opt_ml_price_stamp.wrapping_add(1).max(1); + ml_price_stamp = $self.bt.opt_ml_price_stamp; // Deferred base/seed prices: only reached on a matched seed (see // the declarations above). Assign before the forward DP / seed // paths below read them. @@ -542,7 +541,7 @@ macro_rules! build_optimal_plan_impl_body { let prev_node = unsafe { *nodes.add(pos - 1) }; let lit_len = prev_node.litlen as usize + 1; let lit_price = { - let bt = $self.backend.bt_mut(); + let bt = &mut *$self.bt; BtMatcher::cached_literal_price::( profile, $stats, @@ -593,7 +592,7 @@ macro_rules! build_optimal_plan_impl_body { { if ll1_price < ll0_price { let next_lit_price = { - let bt = $self.backend.bt_mut(); + let bt = &mut *$self.bt; BtMatcher::cached_literal_price::( profile, $stats, @@ -739,7 +738,7 @@ macro_rules! build_optimal_plan_impl_body { // Upstream `ZSTD_optLdm_processMatchCandidate`: the producer // advances past this position and its candidate joins after // the search. - if let Some(ldm) = $self.backend.bt_mut().ldm_process_match_candidate( + if let Some(ldm) = $self.bt.ldm_process_match_candidate( &mut opt_ldm, pos, $current_len - pos, @@ -1306,13 +1305,11 @@ impl HcMatchGenerator { // The scalar arm of `run_main_loop!` wraps a safe `*_scalar` call in // `unsafe` (the SIMD arms need it); allow the redundant block here. #[allow(unused_unsafe)] - pub(crate) fn start_matching_optimal( + pub(crate) fn start_matching_optimal( &mut self, mut handle_sequence: impl FnMut(Sequence), ) { self.table.ensure_tables(); - // Borrowed-aware: owned → last committed chunk; borrowed → staged - // in-place block range. let (current_abs_start, current_len) = self.table.current_block_range(); if current_len == 0 { return; @@ -1335,7 +1332,7 @@ impl HcMatchGenerator { .skip_insert_until_abs .max(self.table.history_abs_start); self.table - .backfill_boundary_positions(current_abs_start, current_abs_end); + .backfill_boundary_positions::(current_abs_start, current_abs_end); self.table.next_to_update3 = hash3_start_cursor; // Take the five DP scratch buffers ONCE for both the seed pass and the @@ -1344,7 +1341,12 @@ impl HcMatchGenerator { // separate take + restore round-trip. let mut plan_buffers = self.take_optimal_plan_buffers(); if self.should_run_btultra2_seed_pass::(current_len) { - self.run_btultra2_seed_pass(current, current_abs_start, current_len, &mut plan_buffers); + self.run_btultra2_seed_pass::( + current, + current_abs_start, + current_len, + &mut plan_buffers, + ); // The seed pass pushes the stored-index offset past everything it // inserted, so the block has to be cleared again for the main pass // against the offset it leaves behind. @@ -1363,16 +1365,15 @@ impl HcMatchGenerator { // The DP bodies read the strategy's `FAVOR_SMALL_OFFSETS` const directly; // verify the runtime profile (built from the same strategy) agrees. debug_assert_eq!(profile.favor_small_offsets, S::FAVOR_SMALL_OFFSETS); - let mut opt_state = - core::mem::replace(&mut self.backend.bt_mut().opt_state, HcOptState::new()); + let mut opt_state = core::mem::replace(&mut self.bt.opt_state, HcOptState::new()); opt_state.rescale_freqs(current, S::ACCURATE_PRICE); - let mut best_plan = core::mem::take(&mut self.backend.bt_mut().opt_segment_plan_scratch); + let mut best_plan = core::mem::take(&mut self.bt.opt_segment_plan_scratch); best_plan.clear(); let plan_reps = self.table.offset_hist; let (cursor, plan_litlen) = self.table.opt_start_cursor_and_litlen(current_abs_start); // Frame-constant LDM presence, resolved once per block (not per segment // and not in the DP hot loop): drives the HAS_LDM const-generic dispatch. - let has_ldm = !self.backend.bt_mut().ldm_sequences.is_empty(); + let has_ldm = !self.bt.ldm_sequences.is_empty(); // Resolve the SIMD tier ONCE here: the whole block then runs under one // kernel-monomorphized pass (`run_optimal_block_`) instead of // hitting select_kernel()'s OnceLock atomic + a CPU-tier match per @@ -1498,7 +1499,7 @@ impl HcMatchGenerator { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match self.table.kernel { + crate::encoding::fastpath::tier_match!(K { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => run_main_loop!(run_optimal_block_avx2_bmi2), #[cfg(feature = "kernel-sse")] @@ -1506,7 +1507,7 @@ impl HcMatchGenerator { #[cfg(feature = "kernel-sse")] FastpathKernel::Sse42 => run_main_loop!(run_optimal_block_sse42), FastpathKernel::Scalar => run_main_loop!(run_optimal_block_scalar), - } + }) } #[cfg(all( target_arch = "wasm32", @@ -1537,17 +1538,14 @@ impl HcMatchGenerator { self.table .emit_optimal_plan(current_len, &best_plan, &mut handle_sequence); best_plan.clear(); - self.backend.bt_mut().opt_segment_plan_scratch = best_plan; - let _ = self - .backend - .bt_mut() - .finish_optimal_plan(plan_buffers, (0, [0; 3], 0, 0)); - self.backend.bt_mut().opt_state = opt_state; + self.bt.opt_segment_plan_scratch = best_plan; + let _ = self.bt.finish_optimal_plan(plan_buffers, (0, [0; 3], 0, 0)); + self.bt.opt_state = opt_state; } // The scalar arm of `run_seed_loop!` wraps a safe call in `unsafe`. #[allow(unused_unsafe)] - fn run_btultra2_seed_pass( + fn run_btultra2_seed_pass( &mut self, current: &[u8], current_abs_start: usize, @@ -1567,14 +1565,13 @@ impl HcMatchGenerator { // Same per-block clamp the main pass does; the DP body expects it done. seed_profile.sufficient_match_len = self.hc.sufficient_match_len_for_pass(seed_profile); debug_assert_eq!(seed_profile.favor_small_offsets, S::FAVOR_SMALL_OFFSETS); - let mut opt_state = - core::mem::replace(&mut self.backend.bt_mut().opt_state, HcOptState::new()); + let mut opt_state = core::mem::replace(&mut self.bt.opt_state, HcOptState::new()); opt_state.rescale_freqs(current, S::ACCURATE_PRICE); let seed_reps = self.table.offset_hist; let (cursor, seed_litlen) = self.table.opt_start_cursor_and_litlen(current_abs_start); - let mut seed_plan = core::mem::take(&mut self.backend.bt_mut().opt_seed_plan_scratch); + let mut seed_plan = core::mem::take(&mut self.bt.opt_seed_plan_scratch); seed_plan.clear(); - let has_ldm = !self.backend.bt_mut().ldm_sequences.is_empty(); + let has_ldm = !self.bt.ldm_sequences.is_empty(); // SIMD tier resolved ONCE (see start_matching_optimal): the seed pass // runs as one kernel-monomorphized block pass. It keeps the statistics // and drops the plan (`KEEP_PLAN = false`). @@ -1699,7 +1696,7 @@ impl HcMatchGenerator { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match self.table.kernel { + crate::encoding::fastpath::tier_match!(K { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => run_seed_loop!(run_optimal_block_avx2_bmi2), #[cfg(feature = "kernel-sse")] @@ -1707,7 +1704,7 @@ impl HcMatchGenerator { #[cfg(feature = "kernel-sse")] FastpathKernel::Sse42 => run_seed_loop!(run_optimal_block_sse42), FastpathKernel::Scalar => run_seed_loop!(run_optimal_block_scalar), - } + }) } #[cfg(all( target_arch = "wasm32", @@ -1735,11 +1732,11 @@ impl HcMatchGenerator { run_seed_loop!(run_optimal_block_scalar); } seed_plan.clear(); - self.backend.bt_mut().opt_seed_plan_scratch = seed_plan; + self.bt.opt_seed_plan_scratch = seed_plan; // The DP scratch buffers are owned by the caller (start_matching_optimal) // and shared across both passes, so the seed pass leaves them in place // (`plan_buffers` is a borrow) instead of restoring them here. - self.backend.bt_mut().opt_state = opt_state; + self.bt.opt_state = opt_state; // Upstream zstd initStats_ultra keeps the collected entropy statistics but // invalidates the first-pass matchfinder history before the real pass. @@ -1767,7 +1764,7 @@ impl HcMatchGenerator { /// literal). Restore with [`BtMatcher::finish_optimal_plan`] after the /// per-segment loop. fn take_optimal_plan_buffers(&mut self) -> HcOptimalPlanBuffers { - let bt = self.backend.bt_mut(); + let bt = &mut *self.bt; let mut nodes = core::mem::take(&mut bt.opt_nodes_scratch); let mut node_prices = core::mem::take(&mut bt.opt_node_prices_scratch); let mut candidates = core::mem::take(&mut bt.opt_candidates_scratch); @@ -2085,7 +2082,7 @@ impl HcMatchGenerator { } #[cfg(test)] - pub(crate) fn collect_optimal_candidates( + pub(crate) fn collect_optimal_candidates( &mut self, abs_pos: usize, current_abs_end: usize, @@ -2115,7 +2112,7 @@ impl HcMatchGenerator { macro_rules! search { ($strategy:ty) => { match self.table.search_mls { - 3 => self.collect_optimal_candidates_initialized::<$strategy, 3>( + 3 => self.collect_optimal_candidates_initialized::<$strategy, K, 3>( abs_pos, current_abs_end, sufficient_match_len, @@ -2123,7 +2120,7 @@ impl HcMatchGenerator { ll0, out, ), - 5 => self.collect_optimal_candidates_initialized::<$strategy, 5>( + 5 => self.collect_optimal_candidates_initialized::<$strategy, K, 5>( abs_pos, current_abs_end, sufficient_match_len, @@ -2131,7 +2128,7 @@ impl HcMatchGenerator { ll0, out, ), - 6 => self.collect_optimal_candidates_initialized::<$strategy, 6>( + 6 => self.collect_optimal_candidates_initialized::<$strategy, K, 6>( abs_pos, current_abs_end, sufficient_match_len, @@ -2139,7 +2136,7 @@ impl HcMatchGenerator { ll0, out, ), - _ => self.collect_optimal_candidates_initialized::<$strategy, 4>( + _ => self.collect_optimal_candidates_initialized::<$strategy, K, 4>( abs_pos, current_abs_end, sufficient_match_len, @@ -2196,6 +2193,7 @@ impl HcMatchGenerator { #[inline(always)] pub(crate) fn collect_optimal_candidates_initialized< S: crate::encoding::strategy::Strategy, + K: EncodeTier, const MLS: usize, >( &mut self, @@ -2224,7 +2222,7 @@ impl HcMatchGenerator { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match self.table.kernel { + crate::encoding::fastpath::tier_match!(K { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { self.collect_optimal_candidates_initialized_avx2_bmi2::( @@ -2267,7 +2265,7 @@ impl HcMatchGenerator { ll0, out, ), - } + }) } // wasm resolves `simd128` at compile time (no runtime detection), so // the tier comes from `cfg`, not from `self.table.kernel`. diff --git a/zstd/src/encoding/lazy_parse.rs b/zstd/src/encoding/lazy_parse.rs index 15428ab8f..ad47ae139 100644 --- a/zstd/src/encoding/lazy_parse.rs +++ b/zstd/src/encoding/lazy_parse.rs @@ -1,19 +1,13 @@ -//! Shared, C-faithful lazy-parse decision logic. +//! The lazy-parse decision as a standalone macro, for the unit tests that probe +//! the row search outside the lazy body. //! //! Upstream zstd has a SINGLE lazy parser — `ZSTD_compressBlock_lazy_generic` //! (`lib/compress/zstd_lazy.c`) — parameterized by the search method -//! (hash-chain / binary-tree / row-hash) and the lookahead `depth`. The match -//! finders differ per strategy; the PARSE DECISION (how a candidate at the -//! current position is weighed against one a byte or two ahead) is identical -//! across all of them. -//! -//! This module hosts that one decision so the per-strategy matchers -//! (HC / Row / dfast) share it instead of each carrying a divergent copy. The -//! register-hungry part is the search kernel, which stays specialized per -//! strategy and is injected here as a closure — the decision itself is a -//! handful of comparisons per position, so sharing it adds no spill pressure. +//! (hash-chain / binary-tree / row-hash) and the lookahead `depth`; the PARSE +//! DECISION (how a candidate at the current position is weighed against one a +//! byte or two ahead) is identical across all of them. -/// C-faithful lazy commit/defer decision, ONE source for every strategy. +/// C-faithful lazy commit/defer decision. /// /// Expanded INLINE at the call site (like upstream's single /// `FORCE_INLINE_TEMPLATE ZSTD_compressBlock_lazy_generic`, zstd_lazy.c:1516): diff --git a/zstd/src/encoding/ldm/mod.rs b/zstd/src/encoding/ldm/mod.rs index 07179b8e9..e375a6dc5 100644 --- a/zstd/src/encoding/ldm/mod.rs +++ b/zstd/src/encoding/ldm/mod.rs @@ -238,7 +238,7 @@ impl LdmProducer { /// The producer covers the **prefix-only** path; the /// two-segment `extDict` variant is documented and deferred /// in [`search`]. - pub(crate) fn generate_into( + pub(crate) fn generate_into( &mut self, live_history: &[u8], history_abs_start: usize, @@ -361,7 +361,7 @@ impl LdmProducer { // mechanism becomes load-bearing only when the // caller drives multiple compress_block windows // through the same producer. - let best = find_best_match( + let best = find_best_match::( &self.hash_table, hash_id, checksum, diff --git a/zstd/src/encoding/ldm/search.rs b/zstd/src/encoding/ldm/search.rs index 782a4c3d4..2b1b43c31 100644 --- a/zstd/src/encoding/ldm/search.rs +++ b/zstd/src/encoding/ldm/search.rs @@ -165,7 +165,7 @@ pub(crate) struct FindBestMatchInputs<'a> { /// [`LdmHashTable::bucket_mask`]. All positions in /// [`FindBestMatchInputs`] are absolute stream coordinates; the /// returned [`LdmMatch::match_pos`] is also absolute. -pub(crate) fn find_best_match( +pub(crate) fn find_best_match( table: &LdmHashTable, hash_id: u32, checksum: u32, @@ -252,7 +252,7 @@ pub(crate) fn find_best_match( // history before the current block are legitimate (upstream zstd // `pMatch < iend` is automatically true when the entry's // absolute offset is below `iend_abs`). - let forward_len = common_prefix_len( + let forward_len = common_prefix_len::( &live_history[split_idx..split_idx_end], &live_history[match_idx..], ); diff --git a/zstd/src/encoding/ldm/search/tests.rs b/zstd/src/encoding/ldm/search/tests.rs index 592de03ae..0412e8dd1 100644 --- a/zstd/src/encoding/ldm/search/tests.rs +++ b/zstd/src/encoding/ldm/search/tests.rs @@ -7,6 +7,18 @@ fn fresh_table() -> LdmHashTable { LdmHashTable::new(4, 2) } +/// [`find_best_match`] on this CPU's kernel tier. +fn find_best( + table: &LdmHashTable, + hash_id: u32, + checksum: u32, + inputs: FindBestMatchInputs<'_>, +) -> Option { + crate::encoding::fastpath::with_host_tier!( + K => find_best_match::(table, hash_id, checksum, inputs) + ) +} + /// `count_backwards_match` honours both lower bounds and /// stops on the first mismatch. Fixture: "XXXabc" matches /// "YYYabc" with 3 backward bytes from offset 3 in each. @@ -47,7 +59,7 @@ fn find_best_match_returns_none_on_checksum_mismatch() { let mut table = fresh_table(); table.insert_absolute(1, 4, 0x1111_1111); let history = b"abcdefghabcdefgh"; - let m = find_best_match( + let m = find_best( &table, 1, 0xDEAD_BEEF, @@ -77,7 +89,7 @@ fn find_best_match_rejects_stale_entries() { // → rejected. (At lowest_index_abs = 4 the entry would // exactly meet the floor and survive — that's the edge // case the inclusive bound is designed to preserve.) - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, @@ -108,7 +120,7 @@ fn find_best_match_picks_longest_combined_match() { let history = b"PPPPabcdefghabcdefgh"; // split at position 12, anchor at 12 → no backward room. // The forward count should match 8 bytes ("abcdefgh"). - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, @@ -141,7 +153,7 @@ fn find_best_match_extends_backwards_into_pre_split_bytes() { // split at 12 (start of second "abcdefgh"), anchor at 10 // → backward up to 2 bytes ("XY" at positions 10..12 vs // 0..2). Forward count: 8 bytes ("abcdefgh"). - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, @@ -178,7 +190,7 @@ fn find_best_match_prefers_longer_total_across_slots() { // offset 8 ("abcdefgh") gives 8 bytes forward; match at // offset 4 ("abcdabcd...") gives only 4 bytes forward // because the 5th byte ('a' vs 'e') mismatches. - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, @@ -210,7 +222,7 @@ fn find_best_match_rejects_entries_at_or_past_split() { // direct caller / test fixture / future extDict variant. table.insert_absolute(1, 12, 0xCAFE); let history = b"PPPPabcdefghabcdefgh"; - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, @@ -230,7 +242,7 @@ fn find_best_match_rejects_entries_at_or_past_split() { // must be rejected. let mut table_after = fresh_table(); table_after.insert_absolute(1, 16, 0xCAFE); - let m_after = find_best_match( + let m_after = find_best( &table_after, 1, 0xCAFE, @@ -261,7 +273,7 @@ fn find_best_match_forward_count_is_bounded_by_iend_abs() { // iend_abs = 16 (4 bytes past the split) the match must // cap at 4. let history = b"PPPPabcdefghabcdefgh"; - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, @@ -291,7 +303,7 @@ fn find_best_match_filters_short_forward_matches() { let history = b"PPPPabXXXXXXab"; // 2-byte forward match from split=12 vs match=4, but // min_match_length = 4 → rejected. - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, diff --git a/zstd/src/encoding/ldm/tests.rs b/zstd/src/encoding/ldm/tests.rs index cdeea4b8d..510bbda9d 100644 --- a/zstd/src/encoding/ldm/tests.rs +++ b/zstd/src/encoding/ldm/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::encoding::fastpath::with_host_tier; /// `LdmParams::adjust_for` must derive a representative /// upstream zstd-btultra2 parameter set at window_log=27. Checked via @@ -45,7 +46,7 @@ fn clear_resets_rolling_hash_state() { let mut out = Vec::new(); // Feed a non-empty chunk so the rolling hash advances. let data = [0xAAu8; 256]; - producer.generate_into(&data, 0, 0, data.len(), &mut out); + with_host_tier!(K => producer.generate_into::(&data, 0, 0, data.len(), &mut out)); let advanced = producer.hash_state.rolling; assert_ne!( advanced, @@ -106,7 +107,7 @@ fn generate_into_emits_long_range_match_on_repeated_payload() { history.extend_from_slice(&payload); let mut out = Vec::new(); - producer.generate_into(&history, 0, 0, history.len(), &mut out); + with_host_tier!(K => producer.generate_into::(&history, 0, 0, history.len(), &mut out)); assert!( !out.is_empty(), @@ -199,7 +200,7 @@ fn generate_into_preserves_bucket_entries_across_history_slide() { // populates with absolute offsets inside payload #1. let mut out1 = Vec::new(); let split_at = PAYLOAD + GAP_A; - producer.generate_into(&frame[..split_at], 0, 0, split_at, &mut out1); + with_host_tier!(K => producer.generate_into::(&frame[..split_at], 0, 0, split_at, &mut out1)); // Call 2: simulate a window slide. The encoder retired // the leading `eviction` bytes from the live history; @@ -215,13 +216,13 @@ fn generate_into_preserves_bucket_entries_across_history_slide() { let block_end_abs = block_start_abs + PAYLOAD; let mut out2 = Vec::new(); - producer.generate_into( + with_host_tier!(K => producer.generate_into::( live, history_abs_start, block_start_abs, block_end_abs, &mut out2, - ); + )); // The long-range match must still fire — entries from // call 1 in the surviving tail of payload #1 @@ -288,7 +289,7 @@ fn generate_into_empty_range_is_noop() { let mut out = Vec::new(); let data = [0u8; 128]; let pre = producer.hash_state.rolling; - producer.generate_into(&data, 0, 64, 64, &mut out); + with_host_tier!(K => producer.generate_into::(&data, 0, 64, 64, &mut out)); assert!(out.is_empty()); assert_eq!(producer.hash_state.rolling, pre); } diff --git a/zstd/src/encoding/levels/config.rs b/zstd/src/encoding/levels/config.rs index c4d334ad9..fdf99e406 100644 --- a/zstd/src/encoding/levels/config.rs +++ b/zstd/src/encoding/levels/config.rs @@ -56,6 +56,16 @@ pub(crate) struct RowConfig { /// Upstream `ZSTD_btlazy2`: the lazy parse searches the lazily-sorted /// binary tree (`ZSTD_BtFindBestMatch`) instead of rows / the chain. pub(crate) bt: bool, + /// Rows or the hash chain, when not the tree (`ZSTD_c_useRowMatchFinder`). + pub(crate) match_finder: crate::encoding::parameters::RowMatchFinderMode, +} + +impl RowConfig { + /// Whether a frame without a dictionary searches rows with a `window`-byte + /// window (a dictionary frame takes its dictionary's answer). + pub(crate) const fn searches_rows(&self, window: usize) -> bool { + !self.bt && self.match_finder.searches_rows(window) + } } // Only used as the default HashChain config when the test-only parse×search @@ -93,6 +103,7 @@ pub(crate) const ROW_CONFIG: RowConfig = RowConfig { mls: ROW_MIN_MATCH_LEN, chain_log: ROW_HASH_BITS, bt: false, + match_finder: crate::encoding::parameters::RowMatchFinderMode::Auto, }; // Level-5 greedy is the ONLY strategy routed to the Row backend @@ -119,6 +130,7 @@ pub(crate) const ROW_L5: RowConfig = RowConfig { mls: ROW_MIN_MATCH_LEN, chain_log: 18, bt: false, + match_finder: crate::encoding::parameters::RowMatchFinderMode::Auto, }; /// Per-level Double-Fast hash sizing, mirroring the upstream zstd `clevels.h` columns @@ -242,9 +254,7 @@ impl LevelParams { SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => { self.row.map(|row| (row.hash_bits, row.chain_log)) } - SearchMethod::HashChain | SearchMethod::BinaryTree => { - self.hc.map(|hc| (hc.hash_log, hc.chain_log)) - } + SearchMethod::BinaryTree => self.hc.map(|hc| (hc.hash_log, hc.chain_log)), }; // The reset synthesises the active config when a row lacks it, so a // missing one only means a table-less level: nothing to index. @@ -264,7 +274,6 @@ impl LevelParams { SearchMethod::Fast => self.fast.map(|fast| fast.mls as usize), SearchMethod::DoubleFast => Some(crate::encoding::match_generator::DFAST_MIN_MATCH_LEN), SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => self.row.map(|row| row.mls), - SearchMethod::HashChain => Some(crate::encoding::hc::HC_MIN_MATCH_LEN), SearchMethod::BinaryTree => self.hc.map(|hc| hc.search_mls), }; configured.unwrap_or(crate::encoding::cost_model::HC_FORMAT_MINMATCH) @@ -347,7 +356,7 @@ pub(crate) fn apply_param_overrides( let row = params.row.get_or_insert(ROW_L5); row.bt = matches!(params.search, SearchMethod::BinaryTreeLazy); } - SearchMethod::HashChain | SearchMethod::BinaryTree => { + SearchMethod::BinaryTree => { params.hc.get_or_insert(HC_OVERRIDE_DEFAULT); } } @@ -416,9 +425,10 @@ pub(crate) fn apply_param_overrides( if let Some(min_match) = ov.min_match { row.mls = min_match as usize; } + row.match_finder = ov.row_match_finder; } } - SearchMethod::HashChain | SearchMethod::BinaryTree => { + SearchMethod::BinaryTree => { if let Some(hc) = params.hc.as_mut() { if let Some(hash_log) = ov.hash_log { hc.hash_log = hash_log as usize; @@ -571,13 +581,6 @@ pub(crate) const MAX_FAST_ATTACH_DICT_REGION: usize = 1 << 24; /// fast-loop levels (L3 / Default / L0) carry the dual-probe. pub(crate) const DFAST_ATTACH_DICT_CUTOFF_LOG: u8 = 14; -/// 32 KiB (`2^15`, upstream zstd `attachDictSizeCutoffs[ZSTD_lazy2]`): small / -/// unknown-size inputs ATTACH the dict as a separate hash-chain dms (the dual -/// search in `find_best_match` walks the live input chain + the dms), larger -/// known-size inputs dense-COPY (merge the dict into the live chain and search -/// the one combined chain). -pub(crate) const HC_ATTACH_DICT_CUTOFF_LOG: u8 = 15; - /// BT/optimal attach cutoff for `btlazy2` + `btopt`: 32 KiB (`2^15`, upstream /// zstd `attachDictSizeCutoffs[ZSTD_btlazy2]` == `[ZSTD_btopt]`). Small / /// unknown-size inputs ATTACH the dict as a separate DUBT dms; larger known-size @@ -663,6 +666,7 @@ fn level_params_from_cparams(cp: crate::encoding::cparams::CParams) -> LevelPara chain_log: cp.chain_log as usize, // Upstream `ZSTD_btlazy2` (strategy 6). bt: cp.strategy == 6, + match_finder: crate::encoding::parameters::RowMatchFinderMode::Auto, }; let bt = |tag| LevelParams { strategy_tag: tag, @@ -1137,17 +1141,17 @@ fn workspace_bytes(params: &LevelParams, ldm: usize) -> usize { table_bytes(4, usize::from(d.long_hash_log)) .saturating_add(table_bytes(4, usize::from(d.short_hash_log))) }), - // The lazy backend's chain / tree finders (window <= 2^14, or a - // btlazy2 level) use a plain hash table (`4 << hash_bits`) plus the - // chain / tree table (`4 << chain_log`) instead of the row tables. + // The lazy backend's chain / tree finders use a plain hash table + // (`4 << hash_bits`) plus the chain / tree table (`4 << chain_log`) + // instead of the row tables. SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => params.row.map_or(0, |r| { - if r.bt || params.window_log <= 14 { + if !r.searches_rows(window) { table_bytes(4, r.hash_bits).saturating_add(table_bytes(4, r.chain_log)) } else { table_bytes(4, r.hash_bits).saturating_add(table_bytes(2, r.hash_bits)) } }), - SearchMethod::HashChain | SearchMethod::BinaryTree => params.hc.map_or(0, |h| { + SearchMethod::BinaryTree => params.hc.map_or(0, |h| { let hash3 = if wants_hash3 { table_bytes( 4, @@ -1397,7 +1401,7 @@ pub(crate) fn resolve_level_params_with_dict( params, Some(RowDictPlan { attach, - use_row: uses_row_match_finder(&cdict), + use_row: uses_row_match_finder(&cdict, overrides.row_match_finder), cdict, }), ) diff --git a/zstd/src/encoding/levels/fastest.rs b/zstd/src/encoding/levels/fastest.rs index 910d222b4..53672895e 100644 --- a/zstd/src/encoding/levels/fastest.rs +++ b/zstd/src/encoding/levels/fastest.rs @@ -8,6 +8,7 @@ use crate::{ commit_block_tables, compress_block, compress_block_with_post_split, discard_block_tables, }, + fastpath::EncodeTier, frame_compressor::CompressState, incompressible::{ block_looks_incompressible, block_looks_incompressible_strict, @@ -40,7 +41,7 @@ use alloc::vec::Vec; // emit-fn shape for no readability gain. #[allow(clippy::too_many_arguments)] #[inline] -pub(crate) fn compress_block_encoded( +pub(crate) fn compress_block_encoded( state: &mut CompressState, compression_level: CompressionLevel, last_block: bool, @@ -168,7 +169,7 @@ pub(crate) fn compress_block_encoded( sink.push(precomputed_checksum.expect("checksum is hashed whenever a sink exists")); } state.matcher.commit_filled(block_len); - state.matcher.skip_matching_with_hint(Some(false)); + skip_block::(state, false); record_unsearched_block(state, block_size); let header = BlockHeader { last_block, @@ -189,7 +190,7 @@ pub(crate) fn compress_block_encoded( sink.push(precomputed_checksum.expect("checksum is hashed whenever a sink exists")); } state.matcher.commit_filled(block_len); - state.matcher.skip_matching_with_hint(Some(true)); + skip_block::(state, true); record_unsearched_block(state, block_size); let header = BlockHeader { last_block, @@ -209,7 +210,7 @@ pub(crate) fn compress_block_encoded( // loop so the cardinality matches the decoder's per-block count // exactly. #[cfg(all(feature = "lsm", feature = "hash"))] - compress_block_with_post_split( + compress_block_with_post_split::( state, last_block, output, @@ -217,9 +218,14 @@ pub(crate) fn compress_block_encoded( block_checksums, ); #[cfg(all(feature = "lsm", not(feature = "hash")))] - compress_block_with_post_split(state, last_block, output, block_decompressed_sizes); + compress_block_with_post_split::( + state, + last_block, + output, + block_decompressed_sizes, + ); #[cfg(not(feature = "lsm"))] - compress_block_with_post_split(state, last_block, output); + compress_block_with_post_split::(state, last_block, output); return BlockType::Compressed; } #[cfg(feature = "lsm")] @@ -244,7 +250,7 @@ pub(crate) fn compress_block_encoded( let hdr_off = output.len(); output.extend_from_slice(&[0u8; 3]); let payload_off = output.len(); - let tables = compress_block(state, output); + let tables = compress_block::(state, output); let payload_len = output.len() - payload_off; // Fall back to a raw block when the compressed payload is not // smaller than the source (`payload >= block_size`) or exceeds the @@ -285,6 +291,17 @@ pub(crate) fn compress_block_encoded( } } +/// Index the committed block without searching it, under the frame's tier `K` +/// when the matcher is the built-in one (which still searches it for +/// long-distance matches). +#[inline] +fn skip_block(state: &mut CompressState, incompressible: bool) { + match state.matcher.builtin_driver() { + Some(driver) => driver.skip_matching_with_hint_with::(Some(incompressible)), + None => state.matcher.skip_matching_with_hint(Some(incompressible)), + } +} + /// A block written out unsearched parses as literals alone: one delimiter /// carrying all of it, when the parse is being recorded. #[inline] @@ -311,7 +328,7 @@ fn record_unsearched_block(state: &mut CompressState, block_size: /// strategy, and the borrowed path is gated to the fast one), so it is /// omitted. #[allow(clippy::too_many_arguments)] -pub(crate) fn compress_block_encoded_borrowed( +pub(crate) fn compress_block_encoded_borrowed( state: &mut CompressState, compression_level: CompressionLevel, last_block: bool, @@ -390,7 +407,7 @@ pub(crate) fn compress_block_encoded_borrowed( sink.push(crate::encoding::frame_compressor::xxh64_block_low32(block)); } state.matcher.set_borrowed_block(block_start, block_end); - state.matcher.skip_matching_with_hint(Some(false)); + state.matcher.skip_matching_with_hint_with::(Some(false)); record_unsearched_block(state, block_size); let header = BlockHeader { last_block, @@ -410,7 +427,7 @@ pub(crate) fn compress_block_encoded_borrowed( sink.push(crate::encoding::frame_compressor::xxh64_block_low32(block)); } state.matcher.set_borrowed_block(block_start, block_end); - state.matcher.skip_matching_with_hint(Some(true)); + state.matcher.skip_matching_with_hint_with::(Some(true)); record_unsearched_block(state, block_size); let header = BlockHeader { last_block, @@ -452,7 +469,7 @@ pub(crate) fn compress_block_encoded_borrowed( let hdr_off = output.len(); output.extend_from_slice(&[0u8; 3]); let payload_off = output.len(); - let tables = compress_block(state, output); + let tables = compress_block::(state, output); let payload_len = output.len() - payload_off; if payload_len >= MAX_BLOCK_SIZE as usize || payload_len >= block_size as usize { // Incompressible (compressed payload not smaller than the source, diff --git a/zstd/src/encoding/levels/fastest/tests.rs b/zstd/src/encoding/levels/fastest/tests.rs index c0063c680..e8a3200b6 100644 --- a/zstd/src/encoding/levels/fastest/tests.rs +++ b/zstd/src/encoding/levels/fastest/tests.rs @@ -92,7 +92,7 @@ fn rle_branch_passes_compressible_hint_to_skip_matching() { (block.len(), false) }); - let emitted = compress_block_encoded( + let emitted = crate::encoding::fastpath::with_host_tier!(K => compress_block_encoded::<_, K>( &mut state, CompressionLevel::Fastest, true, @@ -103,7 +103,7 @@ fn rle_branch_passes_compressible_hint_to_skip_matching() { None, #[cfg(all(feature = "lsm", feature = "hash"))] None, - ); + )); assert_eq!(emitted, BlockType::RLE); assert_eq!( diff --git a/zstd/src/encoding/match_generator/dict_prime.rs b/zstd/src/encoding/match_generator/dict_prime.rs index dff5d4bf9..3155295c3 100644 --- a/zstd/src/encoding/match_generator/dict_prime.rs +++ b/zstd/src/encoding/match_generator/dict_prime.rs @@ -66,7 +66,9 @@ impl MatchGeneratorDriver { self.index_dictionary_in_place(); } - pub(super) fn prime_with_dictionary_impl( + /// [`crate::encoding::Matcher::prime_with_dictionary`] compiled for the kernel tier `K`, the one the encoder chose for the + /// frame. + pub(crate) fn prime_with_dictionary_with( &mut self, dict_content: &[u8], offset_hist: [u32; 3], @@ -244,7 +246,7 @@ impl MatchGeneratorDriver { (chunk.len(), false) }); self.commit_filled(chunk.len()); - self.skip_matching_for_dictionary_priming(dict_content.len()); + self.skip_matching_for_dictionary_priming::(dict_content.len()); committed_dict_budget += end - start; start = end; } @@ -299,7 +301,7 @@ impl MatchGeneratorDriver { .saturating_add(granted_retained_budget); } if self.active_backend() == super::super::strategy::BackendTag::HashChain { - // Recompute the lazy-HC attach decision made per-chunk in + // Recompute the attach decision made per-chunk in // `skip_matching_for_dictionary_priming` (stable across the prime — // `reset_size_log` does not change here). // @@ -308,12 +310,12 @@ impl MatchGeneratorDriver { // table whose dimensions differ from the copy-mode live table, so a // cross-mode restore would install mismatched table geometry and the // encoder could search past the frame window (undecodable). The two - // HC modes share identical window geometry: `max_window_size` and the + // modes share identical window geometry: `max_window_size` and the // dictionary limit are both set ABOVE this branch (the same value in - // either mode), and the live chain table dimensions come from the + // either mode), and the live tree dimensions come from the // resolved `params` the key already pins. The modes differ only in - // WHERE the committed dict lives — a single-link `dms` (attach) vs - // merged into the live chain (copy) — both producing valid matches at + // WHERE the committed dict lives — a separate `dms` tree (attach) vs + // merged into the live tree (copy) — both producing valid matches at // in-window offsets. Upstream zstd makes the same observation: attach // (`ZSTD_resetCCtx_byAttachingCDict`) and copy // (`ZSTD_resetCCtx_byCopyingCDict`) both keep the caller's @@ -334,19 +336,16 @@ impl MatchGeneratorDriver { let attach = self.hc_dict_attach_mode(); let table = &mut self.hc_matcher_mut().table; table.set_dictionary_limit_from_primed_bytes(committed_dict_budget); - // Build the dictMatchState over the committed dict (front of history) - // so `find_best_match` dual-probes it with its own compare budget — - // but ONLY in ATTACH mode. BT/optimal attach → DUBT dms; lazy-HC - // attach → single-link hash-chain dms. COPY mode (large known source, - // both BT and lazy-HC) already merged the dict into the live tree / - // chain in `skip_matching_for_dictionary_priming`, so it carries no - // separate dms — drop any stale one. - if !attach { - table.dms.invalidate(); - } else if table.uses_bt { + // Build the dictMatchState tree over the committed dict (front of + // history) so the search dual-probes it with its own compare budget — + // but ONLY in ATTACH mode. COPY mode (large known source) already + // merged the dict into the live tree in + // `skip_matching_for_dictionary_priming`, so it carries no separate + // dms — drop any stale one. + if attach { table.prime_dms_bt(committed_dict_budget); } else { - table.prime_dms_hc(committed_dict_budget); + table.dms.invalidate(); } } // CDict-equivalent: now that every dict chunk is indexed, mark the @@ -415,23 +414,6 @@ impl MatchGeneratorDriver { (MatcherStorage::Simple(live), MatcherStorage::Simple(snap)) => { live.clone_from(snap); } - // Same-variant HC lazy/greedy restore (non-BT): the snapshot keeps - // the full primed hash/chain tables (capture's non-BT full clone), - // so `clone_from` reuses the live history/hash/chain/dms buffers in - // place — upstream zstd reuses the CDict tables rather than reallocating - // them. This is the per-frame allocate+copy+drop that dominated - // small `compress-dict` HC frames (5-7x vs C). BT (`uses_bt`) - // snapshots drop their live tables, so they stay on the realloc - // path below. - (MatcherStorage::HashChain(live), MatcherStorage::HashChain(snap)) - if !snap.table.uses_bt => - { - live.table.clone_from(&snap.table); - live.hc.clone_from(&snap.hc); - live.strategy_tag = snap.strategy_tag; - // backend is `HcBackend::Hc` (zero-sized) for non-BT levels; - // the live one is already correct for this resolved key. - } // Binary tree: the history and, in COPY mode, the tree land in the // rooms the reset laid out in the workspace. An ATTACH-mode // snapshot holds no tree (see `capture_primed_dictionary`), so the @@ -508,17 +490,13 @@ impl MatchGeneratorDriver { // clone only the dict-state (history + `dms` + window/offset/dict-limit), // then move the live tables back — the snapshot keeps just what upstream zstd's // CDict keeps, and `restore_primed_dictionary` re-allocates the zeroed - // live tables. Every other case keeps the dict reachable through the live - // structure, so the snapshot must retain the full tables (full clone): - // lazy-HC attach (it DOES prime a hash-chain `dms`, but the live chain is - // still the search structure, so the tables must travel) and COPY mode for - // BOTH BT and lazy-HC (`dms` invalidated, dict merged into the live tree / - // chain). `uses_bt && dms.is_primed()` is therefore the exact "decoupled" - // signal — true only for the BT attach prime; lazy-HC attach primes `dms` - // too but is intentionally NOT decoupled. + // live tables. COPY mode keeps the dict reachable through the live tree + // (`dms` invalidated, dict merged in), so its snapshot must retain the + // full tables (full clone). `dms.is_primed()` is therefore the exact + // "decoupled" signal. let bt_decoupled = matches!( &self.storage, - MatcherStorage::HashChain(hc) if hc.table.uses_bt && hc.table.dms.is_primed() + MatcherStorage::HashChain(hc) if hc.table.dms.is_primed() ); if bt_decoupled { let MatcherStorage::HashChain(hc) = &mut self.storage else { diff --git a/zstd/src/encoding/match_generator/long_distance.rs b/zstd/src/encoding/match_generator/long_distance.rs index 757cdb171..58cccbe2a 100644 --- a/zstd/src/encoding/match_generator/long_distance.rs +++ b/zstd/src/encoding/match_generator/long_distance.rs @@ -6,6 +6,7 @@ //! storage directly. use super::{MatchGeneratorDriver, Sequence, owned_sink}; +use crate::encoding::fastpath::EncodeTier; impl MatchGeneratorDriver { /// Search the input from where the last search stopped up to `to`, a @@ -17,7 +18,7 @@ impl MatchGeneratorDriver { /// window back from it, or the whole dictionary while the window has not /// yet passed the dictionary's end. Every match then holds an offset valid /// at its last byte, wherever in the range it lies. - fn search_long_distance_to(&mut self, to: usize) { + fn search_long_distance_to(&mut self, to: usize) { let Some(producer) = self.ldm.as_mut() else { return; }; @@ -36,7 +37,7 @@ impl MatchGeneratorDriver { let floor = reach_floor.max(live_start); debug_assert!(floor <= from, "a search range wider than the window"); self.ldm_found.clear(); - producer.generate_into( + producer.generate_into::( &live[floor - live_start..], floor, from, @@ -56,11 +57,11 @@ impl MatchGeneratorDriver { /// The matches of the last committed block, into [`Self::ldm_block`], /// searched first as far as the block reaches. Returns the block's length. - pub(super) fn take_long_distance_block(&mut self) -> usize { + pub(super) fn take_long_distance_block(&mut self) -> usize { let block_len = self.storage.last_block_len(); let block_end = self.ldm_committed_end; self.align_long_distance_store(block_end - block_len); - self.search_long_distance_to(block_end); + self.search_long_distance_to::(block_end); let min_match = self.reset_shape.map_or( crate::encoding::cost_model::HC_FORMAT_MINMATCH, |(params, ..)| params.min_match(), @@ -73,13 +74,17 @@ impl MatchGeneratorDriver { /// See [`crate::encoding::Matcher::settle_block_cut`]: the next `window` /// uncommitted bytes are searched now, and the cut moved out of any long /// match. - pub(super) fn settle_long_distance_cut(&mut self, window: usize, cut: usize) -> usize { + pub(super) fn settle_long_distance_cut( + &mut self, + window: usize, + cut: usize, + ) -> usize { if self.ldm.is_none() { return cut; } let start = self.ldm_committed_end; self.align_long_distance_store(start); - self.search_long_distance_to(start + window); + self.search_long_distance_to::(start + window); self.ldm_store.cut_outside_matches(window, cut) } @@ -93,12 +98,12 @@ impl MatchGeneratorDriver { /// ordinary search. With `produced`, the caller has emptied the optimal /// parser's candidates first, so a piece the producer fails on is never /// matched against an earlier block's long matches. - pub(super) fn start_matching_long_distance( + pub(super) fn start_matching_long_distance( &mut self, handle_sequence: &mut impl FnMut(Sequence), produced: bool, ) -> bool { - let block_len = self.take_long_distance_block(); + let block_len = self.take_long_distance_block::(); if self.ldm_block.is_empty() { return false; } @@ -110,7 +115,7 @@ impl MatchGeneratorDriver { let mut gap = Some(0usize); if seq.lit_length != 0 { self.storage.commit_piece(seq.lit_length); - self.match_gap(produced, &mut owned_sink(handle_sequence, &mut gap)); + self.match_gap::(produced, &mut owned_sink(handle_sequence, &mut gap)); } let gap_literals = gap.expect("the gap sink only ever writes Some"); self.storage.commit_piece(seq.match_length); @@ -127,7 +132,7 @@ impl MatchGeneratorDriver { if tail != 0 { self.storage.commit_piece(tail); let mut forward = None; - self.match_gap(produced, &mut owned_sink(handle_sequence, &mut forward)); + self.match_gap::(produced, &mut owned_sink(handle_sequence, &mut forward)); } self.ldm_block = sequences; self.ldm_split_block = Some(block_len); @@ -136,10 +141,10 @@ impl MatchGeneratorDriver { /// Match the piece just committed between long matches: by the producer /// when `produced` and it gives a parse, by the backend otherwise. - fn match_gap(&mut self, produced: bool, sink: &mut impl FnMut(Sequence)) { - if produced && self.match_produced(sink) { + fn match_gap(&mut self, produced: bool, sink: &mut impl FnMut(Sequence)) { + if produced && self.match_produced::(sink) { return; } - self.start_matching_owned(sink); + self.start_matching_owned::(sink); } } diff --git a/zstd/src/encoding/match_generator/mod.rs b/zstd/src/encoding/match_generator/mod.rs index 1f6aaa224..caa1587d7 100644 --- a/zstd/src/encoding/match_generator/mod.rs +++ b/zstd/src/encoding/match_generator/mod.rs @@ -22,6 +22,7 @@ use super::cost_model::{ #[cfg(test)] use super::cost_model::{HC_BLOCKSIZE_MAX, HC_MAX_LL, HC_MAX_ML, HC_MAX_OFF, HcOptPriceType}; use super::dfast::DfastMatchGenerator; +use super::fastpath::EncodeTier; #[cfg(test)] use super::hc::HC_MIN_MATCH_LEN; #[cfg(test)] @@ -138,10 +139,6 @@ use super::match_table::storage::{HC_PRIME3BYTES, HC_PRIME4BYTES}; // macros / configs / tests keep their unqualified names. #[cfg(test)] use super::match_table::storage::HC_EMPTY; -// HC3_MAX_OFFSET moved to encoding::bt alongside the hash3 candidate -// probe macro that consumes it; the macro references it via the -// fully-qualified `$crate::encoding::bt::HC3_MAX_OFFSET` path so this -// module no longer needs a local import. pub(crate) const HC_SEARCH_DEPTH: usize = 16; // HC_MIN_MATCH_LEN moved to encoding::hc; re-imported here so // existing references compile unchanged. @@ -210,11 +207,8 @@ enum MatcherStorage { /// `SearchMethod::BinaryTreeLazy`. The Row backend itself searches a hash /// chain instead of rows when the resolved window is small. Row(RowMatchGenerator), - /// The hash-chain matcher, for `SearchMethod::HashChain` and - /// `SearchMethod::BinaryTree`; the latter carries the BT-based optimal - /// modes (`btopt` / `btultra` / `btultra2`). The [`HcMatchGenerator`]'s - /// internal [`HcBackend`] discriminator decides whether BT scratch is - /// allocated. + /// The binary-tree matcher of the optimal parsers (`btopt` / `btultra` / + /// `btultra2`), for `SearchMethod::BinaryTree`. HashChain(HcMatchGenerator), } @@ -495,6 +489,10 @@ pub struct MatchGeneratorDriver { /// A parser asked for each block in place of the backends, when one is /// installed (C `ZSTD_registerSequenceProducer`). producer: Option, + /// This CPU's kernel tier, detected once at construction: the tier the + /// [`Matcher`] entry points run under when a caller drives the driver + /// through the trait rather than through the encoder's own tier. + tier: super::fastpath::FastpathKernel, } /// A [`SequenceProducer`] and what to do when it gives no parse. @@ -564,15 +562,11 @@ struct PrimedKey { /// Whether a configured HashChain matcher attaches the dictionary for a frame /// of `size_log` (see [`MatchGeneratorDriver::hc_dict_attach_mode`]). fn hc_attaches_dictionary(hc: &HcMatchGenerator, size_log: Option) -> bool { - let cutoff = if hc.table.uses_bt { - match hc.strategy_tag { - super::strategy::StrategyTag::BtUltra | super::strategy::StrategyTag::BtUltra2 => { - BT_ULTRA_ATTACH_DICT_CUTOFF_LOG - } - _ => BT_OPT_ATTACH_DICT_CUTOFF_LOG, + let cutoff = match hc.strategy_tag { + super::strategy::StrategyTag::BtUltra | super::strategy::StrategyTag::BtUltra2 => { + BT_ULTRA_ATTACH_DICT_CUTOFF_LOG } - } else { - HC_ATTACH_DICT_CUTOFF_LOG + _ => BT_OPT_ATTACH_DICT_CUTOFF_LOG, }; size_log.is_none_or(|log| log <= cutoff) } @@ -799,6 +793,7 @@ impl MatchGeneratorDriver { #[cfg(feature = "ldm")] ldm_split_block: None, producer: None, + tier: super::fastpath::select_kernel(), } } @@ -838,7 +833,10 @@ impl MatchGeneratorDriver { /// piece of one between long matches, indexing it as an unsearched one. /// Returns `false`, having done nothing, when the producer failed and the /// backend is to match it. - fn match_produced(&mut self, handle_sequence: &mut impl FnMut(Sequence)) -> bool { + fn match_produced( + &mut self, + handle_sequence: &mut impl FnMut(Sequence), + ) -> bool { let mut slot = self .producer .take() @@ -871,7 +869,7 @@ impl MatchGeneratorDriver { return false; } // The block's long matches, if any, were taken before it got here. - self.skip_backend(None); + self.skip_backend::(None); if produced { let mut literals = 0usize; for sequence in &slot.parse { @@ -920,13 +918,9 @@ impl MatchGeneratorDriver { } /// Whether the borrowed (no-copy, in-place over-window) scan is - /// implemented for the current backend + search configuration. The - /// HashChain backend serves both the lazy CHAIN parser - /// (`SearchMethod::HashChain`) and the BT/optimal parsers - /// (`SearchMethod::BinaryTree`); only the lazy chain has a borrowed scan - /// so far, so BT/optimal stay on the owned path. + /// implemented for the current backend. pub(crate) fn borrowed_supported(&self) -> bool { - use super::strategy::{BackendTag, SearchMethod}; + use super::strategy::BackendTag; // A producer reads the committed block, which the in-place scan never // commits. if self.producer.is_some() { @@ -934,18 +928,11 @@ impl MatchGeneratorDriver { } match self.active_backend() { BackendTag::Simple | BackendTag::Dfast | BackendTag::Row => true, - // The HashChain backend covers two searches: the lazy CHAIN parser - // (borrowed-capable) and the BINARY-TREE search (btlazy2 L13-15 + - // optimal BtOpt/BtUltra/BtUltra2 L16-22). btlazy2's BT-tree borrowed - // scan is byte-identical to owned (reads via live_history()), so it - // takes the in-place path. The OPTIMAL parsers stay owned: their - // cost-based DP is sensitive to candidate quality, and the borrowed - // continuous-index scan yields slightly different (ratio-worse) - // candidates than the owned evict+rehash scan — borrowed optimal - // both diverged from owned and fell outside the ffi ratio bound. - // Search-aware (not just strategy_tag) so optimal BT can never be - // staged on the borrowed path even via an internal caller. - BackendTag::HashChain => matches!(self.search, SearchMethod::HashChain), + // The optimal parsers stay owned: their cost-based DP is sensitive + // to candidate quality, and a borrowed continuous-index scan yields + // different (ratio-worse) candidates than the owned evict+rehash + // scan. + BackendTag::HashChain => false, } } @@ -1022,9 +1009,9 @@ impl MatchGeneratorDriver { super::strategy::BackendTag::Row => unsafe { self.row_matcher_mut().set_borrowed_window(buffer) }, - super::strategy::BackendTag::HashChain => unsafe { - self.hc_matcher_mut().set_borrowed_window(buffer) - }, + super::strategy::BackendTag::HashChain => { + unreachable!("the optimal parsers have no borrowed scan") + } } } @@ -1035,9 +1022,7 @@ impl MatchGeneratorDriver { super::strategy::BackendTag::Simple => self.simple_mut().clear_borrowed_window(), super::strategy::BackendTag::Dfast => self.dfast_matcher_mut().clear_borrowed_window(), super::strategy::BackendTag::Row => self.row_matcher_mut().clear_borrowed_window(), - super::strategy::BackendTag::HashChain => self.hc_matcher_mut().clear_borrowed_window(), - #[allow(unreachable_patterns)] - _ => {} + super::strategy::BackendTag::HashChain => {} } self.borrowed_pending = None; } @@ -1074,10 +1059,9 @@ impl MatchGeneratorDriver { super::strategy::BackendTag::Row => self .row_matcher_mut() .stage_borrowed_block(block_start, block_end), - super::strategy::BackendTag::HashChain => self - .hc_matcher_mut() - .table - .stage_borrowed_block(block_start, block_end), + super::strategy::BackendTag::HashChain => { + unreachable!("`borrowed_supported` refuses the optimal parsers") + } } } @@ -1272,7 +1256,7 @@ impl MatchGeneratorDriver { .is_none_or(|log| log <= DFAST_ATTACH_DICT_CUTOFF_LOG) } - fn skip_matching_for_dictionary_priming(&mut self, dict_len: usize) { + fn skip_matching_for_dictionary_priming(&mut self, dict_len: usize) { match self.active_backend() { super::strategy::BackendTag::Simple => { // Upstream zstd `ZSTD_shouldAttachDict` mode selection for the Fast @@ -1330,20 +1314,17 @@ impl MatchGeneratorDriver { self.row_matcher_mut().prime_dictionary_current_block(); } super::strategy::BackendTag::HashChain => { - // Lazy-HC AND BT/optimal both follow upstream zstd `ZSTD_shouldAttachDict` - // per-strategy: ATTACH (a separate dms — hash-chain dms for lazy, - // DUBT dms for BT) for small / unknown inputs, COPY (merge the dict - // into the live chain/tree) for large known inputs. ATTACH keeps - // the dict in history but out of the live structure via - // `skip_matching_dict_bt` (the cursor advance is shared by both - // arms); COPY routes through the normal `skip_matching` (its - // `uses_bt` branch fills the live tree, the lazy branch the live - // chain). The dms is built-or-dropped to match in - // `prime_with_dictionary`. + // Upstream zstd `ZSTD_shouldAttachDict` per strategy: ATTACH (a + // separate DUBT dms) for small / unknown inputs, COPY (merge the + // dict into the live tree) for large known inputs. ATTACH keeps + // the dict in history but out of the live tree via + // `skip_matching_dict_bt`; COPY routes through the normal + // `skip_matching`, which fills the live tree. The dms is + // built-or-dropped to match in `prime_with_dictionary`. if self.hc_dict_attach_mode() { self.hc_matcher_mut().table.skip_matching_dict_bt(); } else { - self.hc_matcher_mut().skip_matching(Some(false)); + self.hc_matcher_mut().skip_matching::(Some(false)); } } } @@ -1531,8 +1512,8 @@ impl Matcher for MatchGeneratorDriver { params.search = search; params.lazy_depth = parse.lazy_depth(); // The matrix sweep can pair a level with a backend its native - // row doesn't populate (e.g. greedy L5, which carries only `row`, - // run on HashChain). Synthesize a default config for the + // row doesn't populate (e.g. dfast L3, which carries only `dfast`, + // run on rows). Synthesize a default config for the // overridden backend so its `configure` arm has something to read. use super::strategy::SearchMethod; match search { @@ -1546,7 +1527,7 @@ impl Matcher for MatchGeneratorDriver { let row = params.row.get_or_insert(ROW_CONFIG); row.bt = matches!(search, SearchMethod::BinaryTreeLazy); } - SearchMethod::HashChain | SearchMethod::BinaryTree => { + SearchMethod::BinaryTree => { params.hc.get_or_insert(HC_CONFIG); } } @@ -1997,19 +1978,16 @@ impl Matcher for MatchGeneratorDriver { } MatcherStorage::HashChain(hc) => { hc.table.max_window_size = max_window_size; - hc.hc.lazy_depth = params.lazy_depth; let mut hc_cfg = params.hc.expect("HashChain level row carries an HcConfig"); - // Cap the hash / chain table logs by the hinted window so a small + // Cap the hash / tree table logs by the hinted window so a small // input doesn't allocate the full level's tables (the upstream zstd // `ZSTD_adjustCParams_internal` clamp: `hashLog <= windowLog + 1`, - // and `cycleLog <= windowLog` — `cycleLog == chainLog` for the HC - // finder, `chainLog - 1` for the BT pair table, so `chainLog <= - // windowLog` (+1 for BT)). Ratio-neutral: a hinted window of - // `2^wlog` bytes holds at most `2^wlog` positions, so the slots - // beyond that are never populated — capping only sheds unused - // allocation. Was the source of L10-lazy peak-alloc ~2.15x the - // upstream zstd on a 1 MiB input. Only applied when hinted; an - // unknown-size stream keeps the full level tables. + // and `cycleLog <= windowLog` with `cycleLog == chainLog - 1` for + // the BT pair table, so `chainLog <= windowLog + 1`). + // Ratio-neutral: a hinted window of `2^wlog` bytes holds at most + // `2^wlog` positions, so the slots beyond that are never + // populated — capping only sheds unused allocation. Only applied + // when hinted; an unknown-size stream keeps the full level tables. // Skip for dictionary frames: their `hc_cfg.{hash,chain}_log` // are the CDict's (verbatim when the dictionary is copied // into the live tables, source-adjusted when it is attached); @@ -2017,15 +1995,8 @@ impl Matcher for MatchGeneratorDriver { // dictionary's tables to the small hinted source. if hinted && dict_hint.is_none() { let wlog = hc_hash_bits_for_window(table_window_size); - let uses_bt = matches!( - strategy_tag, - super::strategy::StrategyTag::Btlazy2 - | super::strategy::StrategyTag::BtOpt - | super::strategy::StrategyTag::BtUltra - | super::strategy::StrategyTag::BtUltra2 - ); hc_cfg.hash_log = hc_cfg.hash_log.min(wlog + 1); - hc_cfg.chain_log = hc_cfg.chain_log.min(if uses_bt { wlog + 1 } else { wlog }); + hc_cfg.chain_log = hc_cfg.chain_log.min(wlog + 1); } hc.configure(hc_cfg, strategy_tag, params.window_log); // A copy-mode frame merges the dictionary into the live tables, @@ -2105,7 +2076,12 @@ impl Matcher for MatchGeneratorDriver { #[inline] fn prime_with_dictionary(&mut self, dict_content: &[u8], offset_hist: [u32; 3]) { - self.prime_with_dictionary_impl(dict_content, offset_hist) + // For a caller driving the matcher through the trait; the encoder + // calls `prime_with_dictionary_with` under its frame's tier. + crate::encoding::fastpath::with_encode_tier!( + self.tier, + K => self.prime_with_dictionary_with::(dict_content, offset_hist) + ) } #[inline] @@ -2164,13 +2140,12 @@ impl Matcher for MatchGeneratorDriver { } fn settle_block_cut(&mut self, window: usize, cut: usize) -> usize { - #[cfg(feature = "ldm")] - return self.settle_long_distance_cut(window, cut); - #[cfg(not(feature = "ldm"))] - { - let _ = window; - cut - } + // For a caller driving the matcher through the trait; the encoder + // calls `settle_block_cut_with` under its frame's tier. + crate::encoding::fastpath::with_encode_tier!( + self.tier, + K => self.settle_block_cut_with::(window, cut) + ) } fn get_last_space(&mut self) -> &[u8] { @@ -2261,106 +2236,25 @@ impl Matcher for MatchGeneratorDriver { } fn start_matching(&mut self, mut handle_sequence: impl FnMut(Sequence)) { - // Borrowed one-shot Fast path: if the frame driver staged a - // block range via `set_borrowed_block`, scan it in place against - // the borrowed window instead of the owned committed block. Only - // the Simple backend is instrumented (the gate guarantees it), - // and the stage is consumed so the next block re-stages. - if let Some((block_start, block_end)) = self.borrowed_pending.take() { - // The borrowed scans share their kernels with the owned ones, so - // they report through the same sink type. - let mut forward = None; - let mut sink = owned_sink(&mut handle_sequence, &mut forward); - match self.active_backend() { - super::strategy::BackendTag::Simple => { - let m = self.simple_mut(); - if m.dict_is_attached() { - // Dict-attach borrowed scan: live matches read the - // borrowed input in place, dict matches read the - // committed dict prefix via the 2-segment counter. - m.start_matching_borrowed_dict(block_start, block_end, &mut sink); - } else { - m.start_matching_borrowed(block_start, block_end, &mut sink); - } - } - super::strategy::BackendTag::Dfast => self - .dfast_matcher_mut() - .start_matching_borrowed(block_start, block_end, &mut sink), - super::strategy::BackendTag::Row => { - // Same greedy/lazy parse split as the owned RowHash arm. - let greedy = self.parse == super::strategy::ParseMode::Greedy; - self.row_matcher_mut().start_matching_borrowed( - block_start, - block_end, - greedy, - &mut sink, - ); - } - super::strategy::BackendTag::HashChain => match self.search { - super::strategy::SearchMethod::HashChain => self - .hc_matcher_mut() - .start_matching_lazy_borrowed(block_start, block_end, &mut sink), - // `borrowed_supported()` keeps the optimal parsers on the - // owned path and `set_borrowed_block` asserts it. - other => { - unreachable!("HashChain backend with unexpected borrowed search {other:?}") - } - }, - } - return; - } - if self.producer.is_some() { - // A producer stands in for every parser, the optimal one too, so - // its block's long matches are taken outright at any strategy and - // the producer parses the literals between them. Upstream refuses - // the pair instead (zstd.h, block-level sequence producer - // limitations); taking both loses neither. - #[cfg(feature = "ldm")] - if self.ldm.is_some() { - // Where the producer fails with fallback, the backend matches - // what is left without long-match candidates: this block's are - // taken outright, and the optimal parser still holds the last - // block's, whose block-relative positions would name the wrong - // bytes here (its candidates are trusted, not re-checked). - if matches!(self.search, super::strategy::SearchMethod::BinaryTree) { - let MatcherStorage::HashChain(hc) = &mut self.storage else { - unreachable!("the optimal parser runs on the hash-chain backend"); - }; - hc.backend.bt_mut().ldm_sequences.clear(); - } - if self.start_matching_long_distance(&mut handle_sequence, true) { - return; - } - if self.match_produced(&mut handle_sequence) { - return; - } - // The producer failed with fallback on a block with no long - // match, which is already taken: the backend matches it. - let mut gap_literals = None; - self.start_matching_owned(&mut owned_sink(&mut handle_sequence, &mut gap_literals)); - return; - } - if self.match_produced(&mut handle_sequence) { - return; - } - } - #[cfg(feature = "ldm")] - if self.ldm.is_some() { - // The optimal parser weighs long-distance matches as candidates; - // below it they are taken outright (upstream - // `ZSTD_ldm_blockCompress`, `zstd_ldm.c:685-749`). - if matches!(self.search, super::strategy::SearchMethod::BinaryTree) { - self.take_long_distance_block(); - let MatcherStorage::HashChain(hc) = &mut self.storage else { - unreachable!("the optimal parser runs on the hash-chain backend"); - }; - core::mem::swap(&mut self.ldm_block, &mut hc.backend.bt_mut().ldm_sequences); - } else if self.start_matching_long_distance(&mut handle_sequence, false) { - return; - } - } - let mut gap_literals = None; - self.start_matching_owned(&mut owned_sink(&mut handle_sequence, &mut gap_literals)); + // For a caller driving the matcher through the trait itself, such as + // a wrapper that records what it emits. The encoder does not come + // here: it reaches the driver through `builtin_driver` and searches + // under the tier it chose for the frame. Either way the tier was + // detected once, at construction. + // + // The sink is passed on as one `dyn` type, so the search is compiled + // once per tier for every such caller rather than once per caller's + // closure; the encoder's own path keeps its sink's concrete type. + let sink: &mut dyn FnMut(Sequence) = &mut handle_sequence; + crate::encoding::fastpath::with_encode_tier!( + self.tier, + K => self.start_matching_with::(&mut *sink) + ) + } + + #[inline] + fn builtin_driver(&mut self) -> Option<&mut MatchGeneratorDriver> { + Some(self) } fn skip_matching(&mut self) { @@ -2368,6 +2262,22 @@ impl Matcher for MatchGeneratorDriver { } fn skip_matching_with_hint(&mut self, incompressible_hint: Option) { + // For a caller driving the matcher through the trait; the encoder + // calls `skip_matching_with_hint_with` under its frame's tier. + crate::encoding::fastpath::with_encode_tier!( + self.tier, + K => self.skip_matching_with_hint_with::(incompressible_hint) + ) + } +} + +impl MatchGeneratorDriver { + /// [`Matcher::skip_matching_with_hint`] compiled for the kernel tier `K`: + /// the block is still searched for long-distance matches. + pub(crate) fn skip_matching_with_hint_with( + &mut self, + incompressible_hint: Option, + ) { // Borrowed one-shot Fast path: a staged block range routes to the // borrowed skip (records the range for `get_last_space`, primes // hashes on the dict-priming hint) with no owned-history append @@ -2387,9 +2297,9 @@ impl Matcher for MatchGeneratorDriver { block_end, incompressible_hint, ), - super::strategy::BackendTag::HashChain => self - .hc_matcher_mut() - .skip_matching_borrowed(block_start, block_end, incompressible_hint), + super::strategy::BackendTag::HashChain => { + unreachable!("`borrowed_supported` refuses the optimal parsers") + } } return; } @@ -2398,16 +2308,16 @@ impl Matcher for MatchGeneratorDriver { // taken past. #[cfg(feature = "ldm")] if self.ldm.is_some() { - self.take_long_distance_block(); + self.take_long_distance_block::(); } - self.skip_backend(incompressible_hint); + self.skip_backend::(incompressible_hint); } } impl MatchGeneratorDriver { /// Index the last committed block (or piece of one) in the backend /// without searching it, leaving the long-distance state alone. - fn skip_backend(&mut self, incompressible_hint: Option) { + fn skip_backend(&mut self, incompressible_hint: Option) { match self.active_backend() { super::strategy::BackendTag::Simple => { self.simple_mut() @@ -2419,9 +2329,9 @@ impl MatchGeneratorDriver { super::strategy::BackendTag::Row => self .row_matcher_mut() .skip_matching_with_hint(incompressible_hint), - super::strategy::BackendTag::HashChain => { - self.hc_matcher_mut().skip_matching(incompressible_hint) - } + super::strategy::BackendTag::HashChain => self + .hc_matcher_mut() + .skip_matching::(incompressible_hint), } } } @@ -2457,9 +2367,128 @@ fn owned_sink<'a, F: FnMut(Sequence)>( } impl MatchGeneratorDriver { + /// [`Matcher::start_matching`] compiled for the kernel tier `K`, the one + /// the encoder chose for the frame. Nothing below asks for a tier. + pub(crate) fn start_matching_with( + &mut self, + mut handle_sequence: impl FnMut(Sequence), + ) { + // Borrowed one-shot Fast path: if the frame driver staged a + // block range via `set_borrowed_block`, scan it in place against + // the borrowed window instead of the owned committed block. Only + // the Simple backend is instrumented (the gate guarantees it), + // and the stage is consumed so the next block re-stages. + if let Some((block_start, block_end)) = self.borrowed_pending.take() { + // The borrowed scans share their kernels with the owned ones, so + // they report through the same sink type. + let mut forward = None; + let mut sink = owned_sink(&mut handle_sequence, &mut forward); + match self.active_backend() { + super::strategy::BackendTag::Simple => { + let m = self.simple_mut(); + if m.dict_is_attached() { + // Dict-attach borrowed scan: live matches read the + // borrowed input in place, dict matches read the + // committed dict prefix via the 2-segment counter. + m.start_matching_borrowed_dict::(block_start, block_end, &mut sink); + } else { + m.start_matching_borrowed(block_start, block_end, &mut sink); + } + } + super::strategy::BackendTag::Dfast => self + .dfast_matcher_mut() + .start_matching_borrowed::(block_start, block_end, &mut sink), + super::strategy::BackendTag::Row => { + // Same greedy/lazy parse split as the owned RowHash arm. + let greedy = self.parse == super::strategy::ParseMode::Greedy; + self.row_matcher_mut().start_matching_borrowed::( + block_start, + block_end, + greedy, + &mut sink, + ); + } + super::strategy::BackendTag::HashChain => { + unreachable!("`borrowed_supported` refuses the optimal parsers") + } + } + return; + } + if self.producer.is_some() { + // A producer stands in for every parser, the optimal one too, so + // its block's long matches are taken outright at any strategy and + // the producer parses the literals between them. Upstream refuses + // the pair instead (zstd.h, block-level sequence producer + // limitations); taking both loses neither. + #[cfg(feature = "ldm")] + if self.ldm.is_some() { + // Where the producer fails with fallback, the backend matches + // what is left without long-match candidates: this block's are + // taken outright, and the optimal parser still holds the last + // block's, whose block-relative positions would name the wrong + // bytes here (its candidates are trusted, not re-checked). + if matches!(self.search, super::strategy::SearchMethod::BinaryTree) { + let MatcherStorage::HashChain(hc) = &mut self.storage else { + unreachable!("the optimal parser runs on the hash-chain backend"); + }; + hc.bt.ldm_sequences.clear(); + } + if self.start_matching_long_distance::(&mut handle_sequence, true) { + return; + } + if self.match_produced::(&mut handle_sequence) { + return; + } + // The producer failed with fallback on a block with no long + // match, which is already taken: the backend matches it. + let mut gap_literals = None; + self.start_matching_owned::(&mut owned_sink( + &mut handle_sequence, + &mut gap_literals, + )); + return; + } + if self.match_produced::(&mut handle_sequence) { + return; + } + } + #[cfg(feature = "ldm")] + if self.ldm.is_some() { + // The optimal parser weighs long-distance matches as candidates; + // below it they are taken outright (upstream + // `ZSTD_ldm_blockCompress`, `zstd_ldm.c:685-749`). + if matches!(self.search, super::strategy::SearchMethod::BinaryTree) { + self.take_long_distance_block::(); + let MatcherStorage::HashChain(hc) = &mut self.storage else { + unreachable!("the optimal parser runs on the hash-chain backend"); + }; + core::mem::swap(&mut self.ldm_block, &mut hc.bt.ldm_sequences); + } else if self.start_matching_long_distance::(&mut handle_sequence, false) { + return; + } + } + let mut gap_literals = None; + self.start_matching_owned::(&mut owned_sink(&mut handle_sequence, &mut gap_literals)); + } + + /// [`Matcher::settle_block_cut`] compiled for the kernel tier `K`. + pub(crate) fn settle_block_cut_with( + &mut self, + window: usize, + cut: usize, + ) -> usize { + #[cfg(feature = "ldm")] + return self.settle_long_distance_cut::(window, cut); + #[cfg(not(feature = "ldm"))] + { + let _ = window; + cut + } + } + /// Match the last committed block of the owned history with the frame's /// backend. - fn start_matching_owned(&mut self, handle_sequence: &mut impl FnMut(Sequence)) { + fn start_matching_owned(&mut self, handle_sequence: &mut impl FnMut(Sequence)) { use super::strategy::{self, StrategyTag}; // Decoupled parse×search dispatch (fires once per block). The // search axis (`self.search`) picks the candidate-finding backend; @@ -2475,7 +2504,8 @@ impl MatchGeneratorDriver { self.simple_mut().start_matching(handle_sequence); } SearchMethod::DoubleFast => { - self.dfast_matcher_mut().start_matching(handle_sequence); + self.dfast_matcher_mut() + .start_matching::(handle_sequence); } SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => { // One upstream `lazy_generic` body for greedy (depth 0: a @@ -2483,17 +2513,16 @@ impl MatchGeneratorDriver { // / lazy2 (`lazy_depth` 1 / 2) and btlazy2 (depth 2 over the // binary-tree finder); the finder (rows / chain / tree) was // resolved at `configure`. - self.row_matcher_mut().start_matching(handle_sequence); - } - SearchMethod::HashChain => { - // Greedy/lazy/lazy2 all flow through the lazy parser; it - // reads `hc.lazy_depth` (0 = greedy commit). - self.hc_matcher_mut().start_matching_lazy(handle_sequence); + self.row_matcher_mut().start_matching::(handle_sequence); } SearchMethod::BinaryTree => match self.strategy_tag { - StrategyTag::BtOpt => self.compress_block::(handle_sequence), - StrategyTag::BtUltra => self.compress_block::(handle_sequence), - StrategyTag::BtUltra2 => self.compress_block::(handle_sequence), + StrategyTag::BtOpt => self.compress_block::(handle_sequence), + StrategyTag::BtUltra => { + self.compress_block::(handle_sequence) + } + StrategyTag::BtUltra2 => { + self.compress_block::(handle_sequence) + } _ => unreachable!( "SearchMethod::BinaryTree requires an optimal strategy tag (BtOpt/BtUltra/BtUltra2)" ), @@ -2508,11 +2537,10 @@ impl MatchGeneratorDriver { /// the concrete opt `S: Strategy` (BtOpt / BtUltra / BtUltra2) off /// `strategy_tag`, so the optimiser keeps the cost-model predicates /// (`S::USE_BT` / `S::USE_HASH3` / `S::ACCURATE_PRICE` / - /// `S::TWO_PASS_SEED`) const-folded per strategy. The non-opt search - /// backends (Fast / DoubleFast / RowHash / HashChain) are dispatched - /// directly off the search axis and never reach this method, so all - /// strategies arriving here are HashChain-backed. - fn compress_block( + /// `S::TWO_PASS_SEED`) const-folded per strategy. The other search + /// backends are dispatched directly off the search axis and never reach + /// this method. + fn compress_block( &mut self, handle_sequence: &mut impl FnMut(Sequence), ) { @@ -2522,33 +2550,10 @@ impl MatchGeneratorDriver { "compress_block only handles the optimal (BT) path" ); self.hc_matcher_mut() - .start_matching_strategy::(handle_sequence); + .start_matching_optimal::(handle_sequence); } } -/// Stage D: backend storage discriminator. -/// -/// HC (lazy / lazy2) modes carry no extra per-frame state beyond the -/// shared `MatchTable` and `HcMatcher` runtime knobs, so the -/// [`HcBackend::Hc`] variant is zero-sized — no BT scratch is -/// allocated. BT-flavoured modes (`btopt` / `btultra` / `btultra2`) -/// hold the full [`super::bt::BtMatcher`] inside the -/// [`HcBackend::Bt`] variant (cost model, optimal-parser scratch -/// arenas, LDM candidate buffer). -/// -/// The discriminator lives next to `parse_mode` so `configure()` can -/// promote between the two on a level change without touching the -/// `MatchTable` storage. -#[derive(Clone)] -pub(crate) enum HcBackend { - /// Lazy / lazy2 modes — no per-frame backend state. - Hc, - /// BT-driven modes — owns the optimal parser's per-frame scratch. - /// Boxed so the enum stays pointer-sized: HC-only matchers pay - /// just the `Box`-niche, not the 4 KiB `BtMatcher` payload. - Bt(alloc::boxed::Box), -} - #[cfg(feature = "bench-internals")] pub(crate) fn level22_block_ranges(data: &[u8]) -> Vec<(usize, usize)> { let mut ranges = Vec::new(); diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index 061bdd7d4..cf1e9b27a 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -3,6 +3,7 @@ //! optimal round-trips, and dictionary-priming behaviour. use super::*; +use crate::encoding::fastpath::{dyn_sink, with_host_tier}; use crate::encoding::test_support::BlockReplay; use alloc::vec::Vec; @@ -45,16 +46,10 @@ impl MatchGeneratorDriver { self.config_override = Some((search, parse)); } - /// Test-only: reset `level` routed onto the lazy HashChain pairing. - /// The lazy band runs on the Row backend in production, so HC-specific - /// behaviour (live-chain dict prime, eviction budget accounting, seed - /// pass gates) is exercised through this override-backed reset. - pub(crate) fn reset_on_hc_lazy(&mut self, level: CompressionLevel) { - self.set_config_override( - super::super::strategy::SearchMethod::HashChain, - super::super::strategy::ParseMode::Lazy2, - ); - self.reset(level); + /// Test-only: the tier the [`Matcher`] entry points run under, in place + /// of the one detected at construction. + pub(crate) fn set_tier(&mut self, tier: crate::encoding::fastpath::FastpathKernel) { + self.tier = tier; } } @@ -90,10 +85,9 @@ fn drive_roundtrip_with_override( out } -/// Phase 1 capability proof: parse and search are decoupled, so a level -/// can run any parse mode on any non-opt search backend. Greedy-on- -/// HashChain and Lazy2-on-RowHash are pairings the legacy `strategy_tag` -/// could not express; both must reconstruct the input exactly. +/// Parse and search are decoupled, so a level can run any parse mode on any +/// non-opt search backend, including one its own row does not configure; the +/// pairings must reconstruct the input exactly. #[test] fn parse_search_matrix_decoupled_roundtrips() { use super::super::strategy::{ParseMode, SearchMethod}; @@ -104,14 +98,14 @@ fn parse_search_matrix_decoupled_roundtrips() { data.extend_from_slice(&i.to_le_bytes()); } - // Greedy parse on the HashChain search backend (legacy: Greedy was - // welded to RowHash). + // A dfast level on the rows: its row carries no row config, so the + // driver synthesizes one. let got = drive_roundtrip_with_override( - CompressionLevel::Level(5), - Some((SearchMethod::HashChain, ParseMode::Greedy)), + CompressionLevel::Level(3), + Some((SearchMethod::RowHash, ParseMode::Greedy)), &data, ); - assert_eq!(got, data, "greedy-on-hashchain diverged"); + assert_eq!(got, data, "greedy-on-rowhash from a dfast level diverged"); // Lazy2 parse on the RowHash search backend (legacy: Lazy was welded // to HashChain). @@ -137,8 +131,7 @@ fn parse_search_matrix_decoupled_roundtrips() { #[cfg(feature = "ldm")] #[test] fn hash_chain_lazy_takes_long_distance_matches() { - use super::super::parameters::CompressionParameters; - use super::super::strategy::{ParseMode, SearchMethod}; + use super::super::parameters::{CompressionParameters, RowMatchFinderMode}; // Six-bit symbols from an LCG's high bits: no repeat the chain finds. let mut state = 0x0569_C4A1u32; @@ -162,15 +155,15 @@ fn hash_chain_lazy_takes_long_distance_matches() { .window_log(22) .hash_log(12) .chain_log(12) + .row_match_finder(RowMatchFinderMode::Disable) .enable_long_distance_matching(long) .build() .unwrap() .overrides(); let mut driver = MatchGeneratorDriver::new(1 << 17, 8); driver.set_param_overrides(Some(overrides)); - driver.set_config_override(SearchMethod::HashChain, ParseMode::Lazy2); driver.reset(CompressionLevel::Level(5)); - assert_eq!(driver.search, SearchMethod::HashChain); + assert!(driver.row_matcher().uses_hash_chain()); let mut out = Vec::with_capacity(data.len()); let mut literals = 0usize; let mut consumed = 0usize; @@ -230,12 +223,12 @@ fn row_mls_knob_gates_matches_and_roundtrips() { let mut out: Vec = Vec::with_capacity(data.len()); let mut shortest_match = usize::MAX; let mut replay = BlockReplay::new(&data); - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if let Sequence::Triple { match_len, .. } = seq { shortest_match = shortest_match.min(match_len); } replay.apply(&mut out, seq); - }); + }))); assert_eq!(out, data, "mls={mls} round-trip diverged"); if shortest_match != usize::MAX { @@ -719,49 +712,25 @@ fn btultra2_seed_pass_initializes_opt_state() { hc.table.commit_input(&data); hc.start_matching(|_| {}); assert!( - hc.backend.bt_mut().opt_state.lit_length_sum > 0, + hc.bt.opt_state.lit_length_sum > 0, "btultra2 first block should seed non-zero sequence statistics" ); assert!( - hc.backend.bt_mut().opt_state.off_code_sum > 0, + hc.bt.opt_state.off_code_sum > 0, "btultra2 first block should seed offset-code statistics" ); } -/// Every per-CPU kernel tier emits a bit-identical sequence stream. Forcing -/// `table.kernel` runs each tier's monomorphized BT-collect / DP wrapper on -/// one machine (only the runtime-selected tier would otherwise execute), which -/// both pins the scalar-vs-SIMD bit-identity invariant and exercises the -/// per-tier wrappers the runtime dispatch leaves cold. x86-only: the aarch64 -/// path dispatches NEON unconditionally, so the cached field is read only in -/// the x86 `cfg` block. -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +/// Every per-CPU kernel tier emits a bit-identical sequence stream. Running +/// the search under each tier the CPU supports executes each monomorphized +/// BT-collect / DP wrapper on one machine (only the detected tier would +/// otherwise run), which pins the scalar-vs-SIMD bit-identity invariant. #[test] fn bt_optimal_all_kernel_tiers_emit_identical_sequences() { use crate::encoding::Sequence; use crate::encoding::fastpath::FastpathKernel; - // Tiers the running CPU can legally execute: each dispatch arm is `unsafe` - // and assumes the tier's target_feature is present. Scalar is always safe; - // the SIMD tiers gate on runtime detection so the test stays valid on any - // x86 box (including a CI runner without AVX2). - // Seeded with the always-present scalar tier; the SIMD entries below are - // feature-gated, so `mut` goes unused in a build with every kernel - // compiled out. - #[allow(unused_mut)] - let mut tiers = alloc::vec![FastpathKernel::Scalar]; - #[cfg(feature = "kernel-sse")] - if std::is_x86_feature_detected!("sse2") { - tiers.push(FastpathKernel::Sse2); - } - #[cfg(feature = "kernel-sse")] - if std::is_x86_feature_detected!("sse4.2") { - tiers.push(FastpathKernel::Sse42); - } - #[cfg(feature = "kernel-avx2")] - if std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("bmi2") { - tiers.push(FastpathKernel::Avx2Bmi2); - } + let tiers = crate::encoding::fastpath::runnable_tiers(); // Mixed-redundancy input so the BT walk, the rep-code probe, and the hash3 // short-match probe all fire (a pure ramp would never exercise the rep path). @@ -769,39 +738,44 @@ fn bt_optimal_all_kernel_tiers_emit_identical_sequences() { .map(|i| ((i * 7 + i / 13) % 67) as u8) .collect(); - let run = |tier: FastpathKernel| -> Vec<(usize, usize, usize)> { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG, - super::super::strategy::StrategyTag::BtUltra2, - 26, - ); - hc.table.commit_input(&data); - hc.table.kernel = tier; - let mut seqs = Vec::new(); - hc.start_matching(|seq| match seq { - Sequence::Triple { - literal_len, - offset, - match_len, - } => seqs.push((literal_len, offset, match_len)), - Sequence::Literals { len } => seqs.push((len, 0, 0)), - }); - seqs - }; + use super::super::strategy::StrategyTag; + for strategy in [ + StrategyTag::BtOpt, + StrategyTag::BtUltra, + StrategyTag::BtUltra2, + ] { + let run = |tier: FastpathKernel| -> Vec<(usize, usize, usize)> { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure(BTULTRA2_HC_CONFIG, strategy, 26); + hc.table.commit_input(&data); + let mut seqs = Vec::new(); + crate::encoding::fastpath::with_encode_tier!( + tier, + K => hc.start_matching_with::(dyn_sink(&mut |seq| match seq { + Sequence::Triple { + literal_len, + offset, + match_len, + } => seqs.push((literal_len, offset, match_len)), + Sequence::Literals { len } => seqs.push((len, 0, 0)), + })) + ); + seqs + }; - let reference = run(tiers[0]); - assert!( - !reference.is_empty(), - "btultra2 should emit sequences on mixed-redundancy input" - ); - for &tier in &tiers[1..] { - assert_eq!( - run(tier), - reference, - "kernel tier {tier:?} diverged from {:?}: scalar/SIMD bit-identity broken", - tiers[0], + let reference = run(tiers[0]); + assert!( + !reference.is_empty(), + "{strategy:?} should emit sequences on mixed-redundancy input" ); + for &tier in &tiers[1..] { + assert_eq!( + run(tier), + reference, + "{strategy:?}: kernel tier {tier:?} diverged from {:?}: scalar/SIMD bit-identity broken", + tiers[0], + ); + } } } @@ -821,36 +795,7 @@ fn bt_optimal_all_kernel_tiers_emit_identical_sequences() { fn dfast_dictionary_all_kernel_tiers_emit_identical_sequences() { use crate::encoding::fastpath::FastpathKernel; - // Only tiers the running CPU may legally execute: each dispatch arm is - // `unsafe` and assumes its target feature is present. - #[allow(unused_mut)] - let mut tiers = alloc::vec![FastpathKernel::Scalar]; - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" - ))] - if std::is_x86_feature_detected!("sse2") { - tiers.push(FastpathKernel::Sse2); - } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" - ))] - if std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("bmi2") { - tiers.push(FastpathKernel::Avx2Bmi2); - } - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - tiers.push(FastpathKernel::Neon); - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - tiers.push(FastpathKernel::Simd128); + let tiers = crate::encoding::fastpath::runnable_tiers(); let dict: Vec = (0..20 * 1024u32) .map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8) @@ -870,22 +815,23 @@ fn dfast_dictionary_all_kernel_tiers_emit_identical_sequences() { driver.set_source_size_hint(block.len() as u64); driver.set_dictionary_size_hint(crate::encoding::DictionarySizes::raw_content(dict.len())); driver.reset(CompressionLevel::Level(3)); - driver.prime_with_dictionary(&dict, [1, 4, 8]); - assert_eq!( - driver.active_backend(), - super::super::strategy::BackendTag::Dfast, - "level 3 with a dictionary must run the dfast backend", - ); - driver.dfast_matcher_mut().kernel = tier; - driver.commit_input(&block); let mut seqs = Vec::new(); - driver.start_matching(|seq| match seq { - Sequence::Triple { - literal_len, - offset, - match_len, - } => seqs.push((literal_len, offset, match_len)), - Sequence::Literals { len } => seqs.push((len, 0, 0)), + crate::encoding::fastpath::with_encode_tier!(tier, K => { + driver.prime_with_dictionary_with::(&dict, [1, 4, 8]); + assert_eq!( + driver.active_backend(), + super::super::strategy::BackendTag::Dfast, + "level 3 with a dictionary must run the dfast backend", + ); + driver.commit_input(&block); + driver.start_matching_with::(dyn_sink(&mut |seq| match seq { + Sequence::Triple { + literal_len, + offset, + match_len, + } => seqs.push((literal_len, offset, match_len)), + Sequence::Literals { len } => seqs.push((len, 0, 0)), + })); }); seqs }; @@ -1056,7 +1002,7 @@ fn dictionary_entropy_seed_initializes_opt_state_from_tables() { let of = crate::fse::fse_encoder::default_of_table(); hc.seed_dictionary_entropy(Some(&huff), Some(&*ll), Some(&*ml), Some(&*of)); - hc.backend.bt_mut().opt_state.rescale_freqs( + hc.bt.opt_state.rescale_freqs( b"abcd", ::ACCURATE_PRICE, ); @@ -1067,22 +1013,15 @@ fn dictionary_entropy_seed_initializes_opt_state_from_tables() { ]; assert_ne!( - hc.backend.bt_mut().opt_state.lit_length_freq, - base_ll_freqs, + hc.bt.opt_state.lit_length_freq, base_ll_freqs, "dictionary entropy should override fallback LL bootstrap frequencies" ); assert!( - hc.backend - .bt_mut() - .opt_state - .match_length_freq - .iter() - .any(|&v| v != 1), + hc.bt.opt_state.match_length_freq.iter().any(|&v| v != 1), "dictionary entropy should seed non-uniform ML frequencies" ); assert_ne!( - hc.backend.bt_mut().opt_state.off_code_freq[0], - 6, + hc.bt.opt_state.off_code_freq[0], 6, "dictionary entropy should override fallback OF bootstrap frequencies" ); } @@ -1101,7 +1040,7 @@ fn dictionary_fse_seed_applies_without_huffman_seed() { let ml = crate::fse::fse_encoder::default_ml_table(); let of = crate::fse::fse_encoder::default_of_table(); hc.seed_dictionary_entropy(None, Some(&*ll), Some(&*ml), Some(&*of)); - hc.backend.bt_mut().opt_state.rescale_freqs( + hc.bt.opt_state.rescale_freqs( b"abcd", ::ACCURATE_PRICE, ); @@ -1111,22 +1050,15 @@ fn dictionary_fse_seed_applies_without_huffman_seed() { 1, 1, 1, 1, 1, 1, ]; assert_ne!( - hc.backend.bt_mut().opt_state.lit_length_freq, - base_ll_freqs, + hc.bt.opt_state.lit_length_freq, base_ll_freqs, "FSE seed should still override LL bootstrap frequencies without huffman seed" ); assert!( - hc.backend - .bt_mut() - .opt_state - .match_length_freq - .iter() - .any(|&v| v != 1), + hc.bt.opt_state.match_length_freq.iter().any(|&v| v != 1), "FSE seed should still seed non-uniform ML frequencies" ); assert_ne!( - hc.backend.bt_mut().opt_state.off_code_freq[0], - 6, + hc.bt.opt_state.off_code_freq[0], 6, "FSE seed should still override OF bootstrap frequencies without huffman seed" ); } @@ -1145,15 +1077,12 @@ fn dictionary_seed_overrides_predef_price_mode_on_tiny_input() { let ml = crate::fse::fse_encoder::default_ml_table(); let of = crate::fse::fse_encoder::default_of_table(); hc.seed_dictionary_entropy(None, Some(&*ll), Some(&*ml), Some(&*of)); - hc.backend.bt_mut().opt_state.rescale_freqs( + hc.bt.opt_state.rescale_freqs( b"abc", ::ACCURATE_PRICE, ); assert!( - matches!( - hc.backend.bt_mut().opt_state.price_type, - HcOptPriceType::Dynamic - ), + matches!(hc.bt.opt_state.price_type, HcOptPriceType::Dynamic), "dictionary-seeded first block should stay in dynamic mode even for tiny src" ); } @@ -1257,7 +1186,7 @@ fn btultra2_seed_pass_disabled_after_stats_initialized() { super::super::strategy::StrategyTag::BtUltra2, 26, ); - hc.backend.bt_mut().opt_state.lit_length_sum = 1; + hc.bt.opt_state.lit_length_sum = 1; assert!( !hc.should_run_btultra2_seed_pass::( HC_PREDEF_THRESHOLD + 32 @@ -1304,7 +1233,7 @@ fn btultra2_seed_pass_runs_alongside_ldm_candidates() { let block_len = HC_PREDEF_THRESHOLD + 64; hc.table.window_size = block_len; hc.table.chunk_lens.push_back(block_len); - hc.backend.bt_mut().ldm_sequences.push(HcRawSeq { + hc.bt.ldm_sequences.push(HcRawSeq { lit_length: 8, offset: 16, match_length: 32, @@ -1383,48 +1312,45 @@ fn dictionary_huffman_seed_ignored_when_literals_uncompressed() { ); } +/// The optimal search probes the repeat offsets in their literal-length order +/// (upstream `ZSTD_insertBtAndGetAllMatches`, zstd_opt.c:657): after a literal +/// `rep[0]` is the first repeat, while at a literal length of zero it is not a +/// repeat at all and `rep[0] - 1` takes its place. #[test] fn hc_repcode_candidates_respect_litlen_dependent_rep_order() { - let mut hc = HcMatchGenerator::new(64); - hc.table.history = b"xxxxxxABCDEFABCDEF".to_vec().into(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - - let abs_pos = 12usize; // points at second "ABCDEF" - let current_abs_end = hc.table.history.len(); - let reps = [6u32, 3u32, 9u32]; - - let mut lit_pos_candidates = Vec::new(); - hc.hc.for_each_repcode_candidate_with_reps( - &hc.table, - abs_pos, - 1, - reps, - current_abs_end, - HC_OPT_MIN_MATCH_LEN, - |c| { - lit_pos_candidates.push(c.offset); - }, - ); + let candidates = |lit_len: usize| { + let mut hc = HcMatchGenerator::new(64); + hc.strategy_tag = crate::encoding::strategy::StrategyTag::BtOpt; + // Only the repeats: no tree compare. + hc.hc.search_depth = 0; + hc.table.search_depth = 0; + // Eight bytes past the searched position, as the parser leaves every + // position it searches. + hc.table.history = b"xxxxxxABCDEFABCDEFzzzzzzzz".to_vec().into(); + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + let abs_pos = 12usize; // the second "ABCDEF" + let current_abs_end = hc.table.history.len(); + let mut out = Vec::new(); + with_host_tier!(K => hc.collect_optimal_candidates::( + abs_pos, + current_abs_end, + usize::MAX / 2, + HcCandidateQuery { + reps: [6, 3, 9], + lit_len, + ldm_candidate: None, + }, + &mut out, + )); + out.iter().map(|c| c.offset).collect::>() + }; assert!( - lit_pos_candidates.contains(&6), + candidates(1).contains(&6), "when lit_len>0, rep0 should be considered and match" ); - - let mut ll0_candidates = Vec::new(); - hc.hc.for_each_repcode_candidate_with_reps( - &hc.table, - abs_pos, - 0, - reps, - current_abs_end, - HC_OPT_MIN_MATCH_LEN, - |c| { - ll0_candidates.push(c.offset); - }, - ); assert!( - !ll0_candidates.contains(&6), + !candidates(0).contains(&6), "when lit_len==0, rep0 is not directly eligible (ll0 semantics)" ); } @@ -1451,7 +1377,7 @@ fn hc_collect_optimal_candidates_keeps_reps_when_chain_depth_zero() { let abs_pos = 6usize; let current_abs_end = hc.table.history.len(); let mut out = Vec::new(); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, current_abs_end, usize::MAX / 2, @@ -1461,7 +1387,7 @@ fn hc_collect_optimal_candidates_keeps_reps_when_chain_depth_zero() { ldm_candidate: None, }, &mut out, - ); + )); assert!( !out.is_empty(), "rep candidates should remain available even when chain depth is zero" @@ -1485,7 +1411,7 @@ fn hc_collect_optimal_candidates_panics_for_non_bt_strategy() { hc.table.history_abs_start = 0; hc.table.ensure_tables(); let mut out = Vec::new(); - hc.collect_optimal_candidates( + hc.collect_optimal_candidates::( 6, hc.table.history.len(), usize::MAX / 2, @@ -1500,7 +1426,7 @@ fn hc_collect_optimal_candidates_panics_for_non_bt_strategy() { #[test] fn hc_collect_optimal_candidates_dispatches_every_bt_strategy() { - use crate::encoding::fastpath::FastpathKernel; + use crate::encoding::fastpath::ScalarTier; use crate::encoding::strategy::StrategyTag; // The public dispatcher must route every BT strategy tag to the collector // AND to the specialization carrying that tag's consts — not just survive. @@ -1518,7 +1444,6 @@ fn hc_collect_optimal_candidates_dispatches_every_bt_strategy() { ] { let mut hc = HcMatchGenerator::new(64); hc.strategy_tag = tag; - hc.table.kernel = FastpathKernel::Scalar; hc.table.history = b"abcQ00000000abcZ00000000".to_vec().into(); hc.table.history_start = 0; hc.table.history_abs_start = 0; @@ -1529,7 +1454,7 @@ fn hc_collect_optimal_candidates_dispatches_every_bt_strategy() { hc.table.search_depth = 8; let abs_pos = 12usize; let mut out = Vec::new(); - hc.collect_optimal_candidates( + hc.collect_optimal_candidates::( abs_pos, hc.table.history.len(), usize::MAX / 2, @@ -1556,30 +1481,13 @@ fn hc_collect_optimal_candidates_dispatches_every_bt_strategy() { /// CPU can run finds the same candidates at every position of a block: the /// scalar fallback and the SIMD kernels agree on the search itself, not only on /// the sequences the parser builds from it. Forcing the tier runs the arms the -/// runtime detection leaves cold on this machine. x86 only, where the tier is a -/// runtime choice. -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +/// runtime detection leaves cold on this machine. #[test] fn hc_collect_optimal_candidates_agree_across_kernel_tiers() { use crate::encoding::fastpath::FastpathKernel; use crate::encoding::strategy::StrategyTag; - // Only tiers the running CPU may execute: each dispatch arm assumes its - // target feature is present. - #[allow(unused_mut)] - let mut tiers = alloc::vec![FastpathKernel::Scalar]; - #[cfg(feature = "kernel-sse")] - if std::is_x86_feature_detected!("sse2") { - tiers.push(FastpathKernel::Sse2); - } - #[cfg(feature = "kernel-sse")] - if std::is_x86_feature_detected!("sse4.2") { - tiers.push(FastpathKernel::Sse42); - } - #[cfg(feature = "kernel-avx2")] - if std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("bmi2") { - tiers.push(FastpathKernel::Avx2Bmi2); - } + let tiers = crate::encoding::fastpath::runnable_tiers(); // Mixed redundancy, so the repeat, hash3 and tree probes all find matches. let data: Vec = (0..4096usize) @@ -1589,20 +1497,22 @@ fn hc_collect_optimal_candidates_agree_across_kernel_tiers() { let mut hc = HcMatchGenerator::new(1 << 20); hc.configure(BTULTRA2_HC_CONFIG, StrategyTag::BtUltra2, 26); hc.table.commit_input(&data); - hc.table.kernel = tier; let mut out = Vec::new(); (0..data.len() - 8) .map(|pos| { - hc.collect_optimal_candidates( - pos, - data.len(), - usize::MAX / 2, - HcCandidateQuery { - reps: [1, 4, 8], - lit_len: 1, - ldm_candidate: None, - }, - &mut out, + crate::encoding::fastpath::with_encode_tier!( + tier, + K => hc.collect_optimal_candidates::( + pos, + data.len(), + usize::MAX / 2, + HcCandidateQuery { + reps: [1, 4, 8], + lit_len: 1, + ldm_candidate: None, + }, + &mut out, + ) ); out.iter().map(|c| (c.offset, c.match_len)).collect() }) @@ -1635,11 +1545,12 @@ fn hc_collect_optimal_candidates_rep_tail_match_skips_chain_probe() { hc.hc.search_depth = 32; let abs_pos = 6usize; hc.table.ensure_tables(); - hc.table.insert_positions(0, abs_pos); - hc.table.search_depth = 32; + let end = hc.table.history.len(); + with_host_tier!(K => hc.table.bt_update_tree_until::(abs_pos, end)); + let mut out = Vec::new(); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, hc.table.history.len(), usize::MAX / 2, @@ -1649,7 +1560,7 @@ fn hc_collect_optimal_candidates_rep_tail_match_skips_chain_probe() { ldm_candidate: None, }, &mut out, - ); + )); assert!( out.iter() @@ -1669,12 +1580,13 @@ fn hc_collect_optimal_candidates_long_chain_match_advances_skip_window() { hc.hc.search_depth = 32; let abs_pos = 9usize; hc.table.ensure_tables(); - hc.table.insert_positions(0, abs_pos); + hc.table.search_depth = 32; + let end = hc.table.history.len(); + with_host_tier!(K => hc.table.bt_update_tree_until::(abs_pos, end)); hc.table.skip_insert_until_abs = 0; - hc.table.search_depth = 32; let mut out = Vec::new(); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, hc.table.history.len(), usize::MAX / 2, @@ -1684,7 +1596,7 @@ fn hc_collect_optimal_candidates_long_chain_match_advances_skip_window() { ldm_candidate: None, }, &mut out, - ); + )); assert!( hc.table.skip_insert_until_abs > abs_pos, @@ -1707,7 +1619,7 @@ fn hc_collect_optimal_candidates_advances_skip_window_on_plain_bt_path() { hc.table.skip_insert_until_abs = 0; hc.table.search_depth = 0; let mut out = Vec::new(); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, hc.table.history.len(), usize::MAX / 2, @@ -1717,7 +1629,7 @@ fn hc_collect_optimal_candidates_advances_skip_window_on_plain_bt_path() { ldm_candidate: None, }, &mut out, - ); + )); assert_eq!( hc.table.skip_insert_until_abs, @@ -1759,7 +1671,7 @@ fn hc_ldm_candidates_are_merged_into_optimal_candidates() { hc.table.search_depth = 0; let mut out = Vec::new(); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, current_abs_end, usize::MAX / 2, @@ -1769,7 +1681,7 @@ fn hc_ldm_candidates_are_merged_into_optimal_candidates() { ldm_candidate: Some(ldm), }, &mut out, - ); + )); assert!( out.iter().any( |candidate| candidate.offset == ldm.offset && candidate.match_len == ldm.match_len @@ -1800,7 +1712,7 @@ fn hc_ldm_candidate_survives_the_search_early_exit() { match_len: 12, }; let mut out = Vec::new(); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, hc.table.history.len(), // Below the repeat match's length, so the repeat probe ends the search. @@ -1811,7 +1723,7 @@ fn hc_ldm_candidate_survives_the_search_early_exit() { ldm_candidate: Some(ldm), }, &mut out, - ); + )); assert!( out.iter().any(|c| c.offset == 10 && c.match_len == 5), "the repeat match that ends the search is kept" @@ -1826,8 +1738,8 @@ fn hc_ldm_candidate_survives_the_search_early_exit() { #[test] fn btultra_and_btultra2_both_keep_dictionary_candidates() { // Routes the BtUltra2 / BtUltra fixture through the production - // `configure()` path so derived state (`hash3_log`, `is_btultra2`, - // `uses_bt`, `backend`) stays consistent — manually flipping the + // `configure()` path so derived state (`hash3_log`, `is_btultra2`) + // stays consistent — manually flipping the // strategy flags here used to leave `hash3_log` / `hash3_table` in // the previous mode's shape and trip the // `Strategy::USE_HASH3 ⇒ hash3_log != 0` debug invariant inside @@ -1858,8 +1770,8 @@ fn btultra_and_btultra2_both_keep_dictionary_candidates() { hc.table.history_abs_start = 0; hc.table.position_base = 0; hc.table.ensure_tables(); - hc.table.insert_positions(0, abs_pos); hc.table.dictionary_limit_abs = Some(64); + // The search threads the tree from here up to `abs_pos` itself. hc.table.skip_insert_until_abs = 0; }; @@ -1869,7 +1781,7 @@ fn btultra_and_btultra2_both_keep_dictionary_candidates() { let mut hc = HcMatchGenerator::new(256); hc.configure(test_config, StrategyTag::BtUltra2, window_log); prepare_history(&mut hc, abs_pos); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, 160, usize::MAX / 2, @@ -1879,7 +1791,7 @@ fn btultra_and_btultra2_both_keep_dictionary_candidates() { ldm_candidate: None, }, &mut out, - ); + )); assert!( out.iter().any(|candidate| candidate.offset >= 32), "btultra2 should retain dictionary candidates on upstream zstd-parity path" @@ -1888,7 +1800,7 @@ fn btultra_and_btultra2_both_keep_dictionary_candidates() { let mut hc = HcMatchGenerator::new(256); hc.configure(test_config, StrategyTag::BtUltra, window_log); prepare_history(&mut hc, abs_pos); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, 160, usize::MAX / 2, @@ -1898,7 +1810,7 @@ fn btultra_and_btultra2_both_keep_dictionary_candidates() { ldm_candidate: None, }, &mut out, - ); + )); assert!( out.iter().any(|candidate| candidate.offset >= 32), "btultra should retain dictionary candidates" @@ -2412,6 +2324,38 @@ fn driver_rows_frame_releases_the_tree_buffer_capacity() { ); } +/// A block skipped through the plain [`Matcher::skip_matching`] is indexed as +/// one skipped with no hint: the next block finds the same matches into it. +#[test] +fn a_plain_skip_indexes_the_block_as_a_skip_without_a_hint() { + let first = b"tenant=demo table=orders region=eu-west payload=".repeat(20); + let run = |plain: bool| { + let mut driver = MatchGeneratorDriver::new(1 << 17, 1); + driver.set_source_size_hint(4 * first.len() as u64); + driver.reset(CompressionLevel::Level(3)); + driver.commit_input(&first); + if plain { + driver.skip_matching(); + } else { + driver.skip_matching_with_hint(None); + } + driver.commit_input(&first); + let mut seqs = Vec::new(); + driver.start_matching(|seq| seqs.push(seq)); + seqs + }; + let reference = run(false); + // From its first byte the block can only match into the skipped one. + assert!( + matches!( + reference.first(), + Some(Sequence::Triple { literal_len: 0, .. }) + ), + "fixture precondition: the second block must match into the skipped one", + ); + assert_eq!(run(true), reference); +} + #[test] fn driver_rows_reset_releases_the_tree_tables() { let mut driver = MatchGeneratorDriver::new(32, 2); @@ -2511,9 +2455,9 @@ fn driver_dictionary_frame_runs_a_strategy_override() { #[test] fn driver_chain_log_override_survives_row_to_hc_fallback() { - // Regression: when a RowHash level is forced onto the HashChain backend - // (resolved window <= 14, upstream `ZSTD_resolveRowMatchFinderMode`), the - // synthesised HC chain table must honour an explicit `chain_log` override. + // Regression: when a RowHash level searches the hash chain (resolved + // window <= 14, upstream `ZSTD_resolveRowMatchFinderMode`), the chain + // table must honour an explicit `chain_log` override. // The RowHash override arm drops `chain_log` (Row has no chain table), so // the synthesis previously replaced the caller's `chain_log` with the upstream zstd // `hashLog - 1`, silently ignoring it on small-window frames. @@ -2595,6 +2539,70 @@ fn driver_small_source_hint_shrinks_row_hash_tables() { ); } +/// `RowMatchFinderMode` replaces the window rule for the greedy / lazy band: +/// `Disable` searches the chain under a wide window, `Enable` the rows under a +/// narrow one, and a btlazy2 level keeps its tree under either (upstream +/// `ZSTD_rowMatchFinderSupported` covers greedy..lazy2 only). +#[test] +fn row_match_finder_mode_picks_the_lazy_finder() { + use crate::encoding::RowMatchFinderMode; + let finder = |level: i32, hint: u64, mode: RowMatchFinderMode| { + let mut driver = MatchGeneratorDriver::new(32, 2); + driver.set_source_size_hint(hint); + driver.set_param_overrides(Some(super::super::parameters::ParamOverrides { + row_match_finder: mode, + ..Default::default() + })); + driver.reset(CompressionLevel::Level(level)); + let row = driver.row_matcher(); + (row.uses_hash_chain(), row.uses_binary_tree()) + }; + let wide = 1 << 20; + let narrow = 1 << 12; + for level in [5, 7] { + assert_eq!( + finder(level, wide, RowMatchFinderMode::Auto), + (false, false) + ); + assert_eq!( + finder(level, wide, RowMatchFinderMode::Disable), + (true, false) + ); + assert_eq!( + finder(level, narrow, RowMatchFinderMode::Auto), + (true, false) + ); + assert_eq!( + finder(level, narrow, RowMatchFinderMode::Enable), + (false, false) + ); + } + assert_eq!(finder(13, wide, RowMatchFinderMode::Disable), (false, true)); + assert_eq!(finder(13, wide, RowMatchFinderMode::Enable), (false, true)); +} + +/// A dictionary frame searches what its dictionary resolved, under the +/// dictionary's mode (upstream `ZSTD_resetCCtx_usingCDict` takes +/// `cdict->useRowMatchFinder`): a dictionary that forces the chain keeps it on +/// a source wide enough for rows. +#[test] +fn row_match_finder_mode_reaches_a_dictionary_frame() { + use crate::encoding::RowMatchFinderMode; + let finder = |mode: RowMatchFinderMode| { + let mut driver = MatchGeneratorDriver::new(32, 2); + driver.set_source_size_hint(1 << 20); + driver.set_dictionary_size_hint(crate::encoding::DictionarySizes::raw_content(1 << 16)); + driver.set_param_overrides(Some(super::super::parameters::ParamOverrides { + row_match_finder: mode, + ..Default::default() + })); + driver.reset(CompressionLevel::Level(7)); + driver.row_matcher().uses_hash_chain() + }; + assert!(!finder(RowMatchFinderMode::Auto)); + assert!(finder(RowMatchFinderMode::Disable)); +} + /// btlazy2 levels run on the lazy (Row) backend with the binary-tree finder, /// wherever the source-size tier puts strategy 6: L13 at tier 0, L10 at the /// <= 16 KiB tier (where L11 is already btopt, an optimal level on the @@ -2643,13 +2651,13 @@ fn row_matches_roundtrip_multi_block_pattern() { matcher.commit_input(&first_block); let mut history = Vec::new(); let mut replay = BlockReplay::new(&first_block); - matcher.start_matching(|seq| replay.apply(&mut history, seq)); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut history, seq)))); assert_eq!(history, first_block); matcher.commit_input(&second_block); let prefix_len = history.len(); let mut replay = BlockReplay::new(&second_block); - matcher.start_matching(|seq| replay.apply(&mut history, seq)); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut history, seq)))); assert_eq!(&history[prefix_len..], second_block.as_slice()); @@ -2658,7 +2666,7 @@ fn row_matches_roundtrip_multi_block_pattern() { matcher.commit_input(&third_block); let third_prefix = history.len(); let mut replay = BlockReplay::new(&third_block); - matcher.start_matching(|seq| replay.apply(&mut history, seq)); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut history, seq)))); assert_eq!(&history[third_prefix..], third_block.as_slice()); } @@ -2672,10 +2680,10 @@ fn row_short_block_emits_literals_only() { let mut saw_triple = false; let mut reconstructed = Vec::new(); let mut replay = BlockReplay::new(b"abcde"); - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { saw_triple |= matches!(seq, Sequence::Triple { .. }); replay.apply(&mut reconstructed, seq); - }); + }))); assert!( !saw_triple, @@ -2688,11 +2696,11 @@ fn row_short_block_emits_literals_only() { // searches (upstream zstd `lazy_generic` `ilimit`). saw_triple = false; matcher.commit_input(b"abcdeabcdeabcdeabcde-padding-past-ilimit"); - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if let Sequence::Triple { .. } = seq { saw_triple = true; } - }); + }))); assert!( saw_triple, "row backend should emit triples on repeated data" @@ -2739,14 +2747,14 @@ fn row_backfills_previous_block_tail_for_cross_boundary_match() { matcher.commit_input(&first_block); let mut reconstructed = Vec::new(); let mut replay = BlockReplay::new(&first_block); - matcher.start_matching(|seq| replay.apply(&mut reconstructed, seq)); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut reconstructed, seq)))); assert_eq!(reconstructed, first_block); matcher.commit_input(&second_block); let mut saw_cross_boundary = false; let prefix_len = reconstructed.len(); let mut replay = BlockReplay::new(&second_block); - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if let Sequence::Triple { literal_len: 0, offset: 3, @@ -2757,7 +2765,7 @@ fn row_backfills_previous_block_tail_for_cross_boundary_match() { saw_cross_boundary = true; } replay.apply(&mut reconstructed, seq); - }); + }))); assert!( saw_cross_boundary, @@ -2893,12 +2901,13 @@ fn source_hint_clamps_driver_slice_size_to_window() { fn driver_best_to_fastest_releases_oversized_hc_tables() { let mut driver = MatchGeneratorDriver::new(32, 2); - // Initialize at Best routed onto HashChain via the test-only override - // (production `Best` sits on level 13, whose native backend differs) — - // allocates large HC tables (4M hash, 2M chain) so the swap below - // exercises the HC drain path this test pins. - driver.reset_on_hc_lazy(CompressionLevel::Best); - assert_eq!(driver.window_size(), (1u64 << 22)); + // An optimal level allocates large tables on the HashChain backend, so the + // swap below exercises the drain path this test pins. + driver.reset(CompressionLevel::Level(19)); + assert_eq!( + driver.active_backend(), + super::super::strategy::BackendTag::HashChain + ); // Feed data so tables are actually allocated via ensure_tables(). driver.commit_input(b"abcabcabcabc"); @@ -2973,7 +2982,7 @@ fn prime_with_dictionary_applies_offset_history_even_when_content_is_empty() { #[test] fn hc_prime_with_empty_dictionary_disables_btultra2_seed_pass() { let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset_on_hc_lazy(CompressionLevel::Better); + driver.reset(CompressionLevel::Level(19)); driver.prime_with_dictionary(&[], [11, 7, 3]); @@ -3095,7 +3104,7 @@ fn a_btultra2_seed_pass_near_the_top_of_the_address_space_parses_like_a_fresh_on #[test] fn hc_prime_with_dictionary_disables_btultra2_seed_pass() { let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset_on_hc_lazy(CompressionLevel::Better); + driver.reset(CompressionLevel::Level(19)); driver.prime_with_dictionary(b"abcdefgh", [1, 4, 8]); @@ -3719,7 +3728,7 @@ fn common_prefix_len_matches_scalar_reference_across_offsets() { let a = &base[start..]; let b = a.to_vec(); assert_eq!( - common_prefix_len(a, &b), + with_host_tier!(K => common_prefix_len::(a, &b)), scalar_reference(a, &b), "equal slices total_len={total_len} start={start}" ); @@ -3732,7 +3741,7 @@ fn common_prefix_len_matches_scalar_reference_across_offsets() { let mut altered = b.clone(); altered[mismatch] ^= 0x5A; assert_eq!( - common_prefix_len(a, &altered), + with_host_tier!(K => common_prefix_len::(a, &altered)), scalar_reference(a, &altered), "total_len={total_len} start={start} mismatch={mismatch}" ); @@ -3743,7 +3752,7 @@ fn common_prefix_len_matches_scalar_reference_across_offsets() { let mut altered = b.clone(); altered[mismatch] ^= 0xA5; assert_eq!( - common_prefix_len(a, &altered), + with_host_tier!(K => common_prefix_len::(a, &altered)), scalar_reference(a, &altered), "tail mismatch total_len={total_len} start={start} mismatch={mismatch}" ); @@ -3754,7 +3763,7 @@ fn common_prefix_len_matches_scalar_reference_across_offsets() { let long = alloc::vec![0xAB; 320]; let shorter = alloc::vec![0xAB; 137]; assert_eq!( - common_prefix_len(&long, &shorter), + with_host_tier!(K => common_prefix_len::(&long, &shorter)), scalar_reference(&long, &shorter) ); } @@ -3897,7 +3906,7 @@ fn row_repcode_skips_candidate_before_history_start() { matcher.history_abs_start = 10; matcher.offset_hist = [3, 0, 0]; - assert!(matcher.repcode_candidate(12, 1).is_none()); + assert!(with_host_tier!(K => matcher.repcode_candidate::(12, 1)).is_none()); } #[test] @@ -3909,7 +3918,7 @@ fn row_repcode_returns_none_when_position_too_close_to_history_end() { matcher.history_abs_start = 0; matcher.offset_hist = [1, 0, 0]; - assert!(matcher.repcode_candidate(4, 1).is_none()); + assert!(with_host_tier!(K => matcher.repcode_candidate::(4, 1)).is_none()); } #[test] @@ -3923,16 +3932,6 @@ fn hc_hash3_position_matches_hash3_formula() { ); } -#[test] -fn hc_hash_position_matches_hash4_formula() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure(HC_CONFIG, super::super::strategy::StrategyTag::Lazy, 22); - let bytes = *b"abcd"; - let read32 = u32::from_le_bytes(bytes); - let expected = ((read32.wrapping_mul(HC_PRIME4BYTES)) >> (32 - hc.table.hash_log)) as usize; - assert_eq!(hc.table.hash_position(&bytes), expected); -} - #[test] fn btultra2_main_hash_uses_hash4_formula() { let mut hc = HcMatchGenerator::new(1 << 20); @@ -3973,8 +3972,9 @@ fn hc_reset_advances_floor_past_prior_frame_entries() { let mut hc = HcMatchGenerator::new(32); hc.table.commit_input(b"abcdeabcde"); hc.table.ensure_tables(); - // Populate real hash / chain entries for the first frame's positions. - hc.table.insert_positions(0, 6); + hc.table.search_depth = 4; + // Populate real hash / tree entries for the first frame's positions. + with_host_tier!(K => hc.table.bt_update_tree_until::(6, 10)); let prev_end = hc.table.history_abs_end(); assert_eq!(prev_end, 10); assert!(hc.table.hash_table().iter().any(|&v| v != HC_EMPTY)); @@ -4043,52 +4043,6 @@ fn deterministic_high_entropy_bytes(seed: u64, len: usize) -> Vec { out } -#[test] -fn hc_sparse_skip_matching_preserves_tail_cross_block_match() { - let mut matcher = HcMatchGenerator::new(1 << 22); - let tail = b"Qz9kLm2Rp"; - let mut first = deterministic_high_entropy_bytes(0xD1B5_4A32_9C77_0E19, 4096); - let tail_start = first.len() - tail.len(); - first[tail_start..].copy_from_slice(tail); - matcher.table.commit_input(&first); - matcher.skip_matching(Some(true)); - - let mut second = tail.to_vec(); - second.extend_from_slice(b"after-tail-literals"); - matcher.table.commit_input(&second); - - let mut first_sequence = None; - matcher.start_matching(|seq| { - if first_sequence.is_some() { - return; - } - first_sequence = Some(match seq { - Sequence::Literals { len } => (len, 0usize, 0usize), - Sequence::Triple { - literal_len, - offset, - match_len, - } => (literal_len, offset, match_len), - }); - }); - - let (literals_len, offset, match_len) = - first_sequence.expect("expected at least one sequence after sparse skip"); - assert_eq!( - literals_len, 0, - "first sequence should start at block boundary" - ); - assert_eq!( - offset, - tail.len(), - "first match should reference previous tail" - ); - assert!( - match_len >= tail.len(), - "tail-aligned cross-block match must be preserved" - ); -} - #[test] fn btultra2_sparse_skip_matching_preserves_tail_cross_block_match() { let mut matcher = HcMatchGenerator::new(1 << 20); @@ -4102,7 +4056,7 @@ fn btultra2_sparse_skip_matching_preserves_tail_cross_block_match() { let tail_start = first.len() - tail.len(); first[tail_start..].copy_from_slice(tail); matcher.table.commit_input(&first); - matcher.skip_matching(Some(true)); + with_host_tier!(K => matcher.skip_matching::(Some(true))); let mut second = tail.to_vec(); second.extend_from_slice(b"after-tail-literals"); @@ -4145,7 +4099,7 @@ fn hc_sparse_skip_matching_does_not_reinsert_sparse_tail_positions() { let mut matcher = HcMatchGenerator::new(1 << 22); let first = deterministic_high_entropy_bytes(0xC2B2_AE3D_27D4_EB4F, 4096); matcher.table.commit_input(&first); - matcher.skip_matching(Some(true)); + with_host_tier!(K => matcher.skip_matching::(Some(true))); let current_len = first.len(); let current_abs_start = @@ -4183,58 +4137,6 @@ fn hc_compact_history_drains_when_threshold_crossed() { assert_eq!(&hc.table.history[..], b"qrstuvwxyz"); } -#[test] -fn hc_insert_position_no_rebase_returns_when_relative_pos_unavailable() { - let mut hc = HcMatchGenerator::new(32); - hc.table.history = b"abcdefghijklmnop".to_vec().into(); - hc.table.history_abs_start = 0; - hc.table.position_base = 1; - hc.table.ensure_tables(); - let before_hash = hc.table.hash_table().to_vec(); - let before_chain = hc.table.chain_table().to_vec(); - - hc.table.insert_position_no_rebase(0); - - assert_eq!(hc.table.hash_table(), before_hash); - assert_eq!(hc.table.chain_table(), before_chain); -} - -#[test] -fn hc_insert_positions_advances_next_to_update3_for_contiguous_range() { - let mut hc = HcMatchGenerator::new(64); - hc.table.history = b"abcdefghijklmnopqrstuvwxyz".to_vec().into(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - hc.table.position_base = 0; - hc.table.ensure_tables(); - hc.table.next_to_update3 = 0; - - hc.table.insert_positions(0, 9); - - assert_eq!( - hc.table.next_to_update3, 9, - "contiguous insert_positions should advance hash3 update cursor" - ); -} - -#[test] -fn hc_insert_positions_with_step_keeps_next_to_update3_cursor_for_sparse_ranges() { - let mut hc = HcMatchGenerator::new(64); - hc.table.history = b"abcdefghijklmnopqrstuvwxyz".to_vec().into(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - hc.table.position_base = 0; - hc.table.ensure_tables(); - hc.table.next_to_update3 = 0; - - hc.table.insert_positions_with_step(0, 16, 4); - - assert_eq!( - hc.table.next_to_update3, 0, - "sparse insert_positions_with_step must not mark skipped positions as hash3-updated" - ); -} - #[test] fn prime_with_dictionary_budget_shrinks_after_dfast_eviction() { let mut driver = MatchGeneratorDriver::new(8, 1); @@ -4266,25 +4168,29 @@ fn prime_with_dictionary_budget_shrinks_after_dfast_eviction() { #[test] fn hc_prime_with_dictionary_preserves_history_for_first_full_block() { - let mut driver = MatchGeneratorDriver::new(8, 1); - // Route onto HashChain explicitly — `Better` resolves to the Row - // backend in production, and this test pins HC dict-prime behaviour. - driver.reset_on_hc_lazy(CompressionLevel::Better); + let mut driver = MatchGeneratorDriver::new(64, 1); + driver.reset(CompressionLevel::Level(19)); - driver.prime_with_dictionary(b"abcdefgh", [1, 4, 8]); + let dict = b"abcdefghijklmnopqrstuvwxyz012345"; + driver.prime_with_dictionary(dict, [1, 4, 8]); - // Repeat the dictionary content so the HC matcher can find it. - // HC_MIN_MATCH_LEN is 5, so an 8-byte match is well above threshold. - driver.commit_input(b"abcdefgh"); + // Repeat the dictionary content so the search can find it, a block long + // enough for the optimal parser to search. + driver.commit_input(dict); + // The parse starts one byte into the block: the attached dictionary leaves + // the prefix starting at the frame's first byte, which the optimal parser + // never searches (upstream zstd_opt.c:1116, `ip += (ip==prefixStart)`). let mut saw_match = false; driver.start_matching(|seq| { if let Sequence::Triple { - literal_len: 0, - offset: 8, + literal_len, + offset, match_len, } = seq - && match_len >= HC_MIN_MATCH_LEN + && literal_len <= 1 + && offset == dict.len() + && literal_len + match_len == dict.len() { saw_match = true; } @@ -4292,14 +4198,14 @@ fn hc_prime_with_dictionary_preserves_history_for_first_full_block() { assert!( saw_match, - "hash-chain backend should match dictionary-primed history in first full block" + "the optimal backend should match dictionary-primed history in first full block" ); } #[test] fn prime_with_dictionary_budget_shrinks_after_hc_eviction() { let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset_on_hc_lazy(CompressionLevel::Better); + driver.reset(CompressionLevel::Level(19)); // Use a small live window so dictionary-primed slices are evicted quickly. driver.hc_matcher_mut().table.max_window_size = 8; driver.reported_window_size = 8; @@ -4337,7 +4243,7 @@ fn resident_reapply_restores_retained_dictionary_budget() { let mut driver = MatchGeneratorDriver::new(1 << 16, 1); let dict = b"abcdefghABCDEFGHijklmnopqrstuvwxyz0123456789"; driver.set_dictionary_size_hint(crate::encoding::DictionarySizes::raw_content(dict.len())); - driver.reset_on_hc_lazy(CompressionLevel::Better); + driver.reset(CompressionLevel::Level(19)); driver.prime_with_dictionary(dict, [1, 4, 8]); let base = driver.reported_window_size; assert!( @@ -4347,7 +4253,7 @@ fn resident_reapply_restores_retained_dictionary_budget() { // Second frame: the reset detects the resident dict and re-borrows it. driver.set_dictionary_size_hint(crate::encoding::DictionarySizes::raw_content(dict.len())); - driver.reset_on_hc_lazy(CompressionLevel::Better); + driver.reset(CompressionLevel::Level(19)); assert!( driver.dictionary_is_resident(), "the second frame must re-borrow the resident dictionary" @@ -4379,7 +4285,7 @@ fn hc_commit_without_eviction_retires_no_dictionary_budget() { // would prematurely retire dictionary budget even when the window is // nowhere near full. let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset_on_hc_lazy(CompressionLevel::Better); + driver.reset(CompressionLevel::Level(19)); // A large live window so a small committed block evicts nothing. driver.hc_matcher_mut().table.max_window_size = 1 << 20; driver.reported_window_size = 1 << 20; @@ -4440,7 +4346,7 @@ fn hc_rebases_positions_after_u32_boundary() { // Simulate a long-running stream where absolute history positions crossed // the u32 range. Before #51 this disabled HC inserts entirely. matcher.table.history_abs_start = history_abs_start; - matcher.skip_matching(None); + with_host_tier!(K => matcher.skip_matching::(None)); assert_eq!( matcher.table.position_base, matcher.table.history_abs_start, "rebase should anchor to the oldest live absolute position" @@ -4499,6 +4405,7 @@ fn hc_rebase_rebuilds_only_inserted_prefix() { let mut matcher = HcMatchGenerator::new(64); matcher.table.commit_input(b"abcdeabcdeabcde"); matcher.table.ensure_tables(); + matcher.table.search_depth = 4; matcher.table.position_base = 0; let history_abs_start: usize = match (u64::from(u32::MAX) + 64).try_into() { Ok(value) => value, @@ -4507,15 +4414,20 @@ fn hc_rebase_rebuilds_only_inserted_prefix() { matcher.table.history_abs_start = history_abs_start; let abs_pos = matcher.table.history_abs_start + 6; + // The tree a walk from the rebased origin builds up to `abs_pos`, with the + // compare the rebase replays on. let mut expected = HcMatchGenerator::new(64); expected.table.commit_input(b"abcdeabcdeabcde"); expected.table.ensure_tables(); + expected.table.search_depth = 4; expected.table.history_abs_start = history_abs_start; expected.table.position_base = expected.table.history_abs_start; + expected.table.skip_insert_until_abs = history_abs_start; expected.table.tables.fill(HC_EMPTY); - for pos in expected.table.history_abs_start..abs_pos { - expected.table.insert_position_no_rebase(pos); - } + let end = history_abs_start + expected.table.window_size; + expected + .table + .bt_update_tree_until::(abs_pos, end); matcher.table.maybe_rebase_positions(abs_pos); @@ -4531,7 +4443,7 @@ fn hc_rebase_rebuilds_only_inserted_prefix() { assert_eq!( matcher.table.chain_table(), expected.table.chain_table(), - "future positions must not be pre-seeded into HC chains during rebase" + "future positions must not be pre-seeded into the tree during rebase" ); } @@ -4567,7 +4479,7 @@ fn dfast_skip_matching_handles_window_eviction() { let mut reconstructed = alloc::vec![7, 8, 9, 10, 11, 12]; let mut replay = BlockReplay::new(&[7, 8, 9, 10, 11, 12]); - matcher.start_matching(|seq| replay.apply(&mut reconstructed, seq)); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut reconstructed, seq)))); assert_eq!(reconstructed, [7, 8, 9, 10, 11, 12, 7, 8, 9, 10, 11, 12]); } @@ -4687,18 +4599,18 @@ fn dfast_inserts_tail_positions_for_next_block_matching() { matcher.commit_input(b"012345bcdea"); let mut history = Vec::new(); let mut replay = BlockReplay::new(b"012345bcdea"); - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { assert!( matches!(seq, Sequence::Literals { .. }), "first block should not match history" ); replay.apply(&mut history, seq); - }); + }))); assert_eq!(history, b"012345bcdea"); matcher.commit_input(b"bcdeabcdeab"); let mut saw_first_sequence = false; - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { assert!(!saw_first_sequence, "expected a single cross-block match"); saw_first_sequence = true; match seq { @@ -4720,7 +4632,7 @@ fn dfast_inserts_tail_positions_for_next_block_matching() { } } } - }); + }))); assert!( saw_first_sequence, @@ -4729,53 +4641,53 @@ fn dfast_inserts_tail_positions_for_next_block_matching() { assert_eq!(history, b"012345bcdeabcdeabcdeab"); } -/// Regression for #49 — locks down `MatchTable::backfill_boundary_positions` -/// for the [`HcMatchGenerator`] lazy path. `backfill_boundary_positions` -/// seeds ONLY the last `< 4` bytes of the previous slice (positions in -/// `[current_abs_start - 3, current_abs_start)`) — the bytes that -/// `insert_position` could not hash at the time because hashing needs -/// 4 bytes of lookahead. The existing 8 MiB window roundtrip test -/// exercises cross-slice behaviour end-to-end, but does not isolate -/// the backfill of those final 1-3 unhashable bytes. +/// Regression for #49 — locks down the boundary backfill of the lazy +/// hash-chain search (the Row backend's chain finder). It seeds the last +/// `< 4` bytes of the previous slice (positions in +/// `[current_abs_start - 3, current_abs_start)`) — the bytes that could not be +/// hashed at the time because hashing needs 4 bytes of lookahead. The +/// existing 8 MiB window roundtrip test exercises cross-slice behaviour +/// end-to-end, but does not isolate the backfill of those final 1-3 +/// unhashable bytes. /// /// Fixture is built so the cross-block match's candidate position /// MUST lie in `[block_1_end - 3, block_1_end)`: /// -/// - Block 1 = `b"PQRSTBCD"` (8 bytes). Block 1's `start_matching` -/// hashes positions 0..=4 (each has 4 bytes of forward context); -/// positions 5/6/7 are the unhashable tail. +/// - Block 1 = `b"PQRSTBCD"` (8 bytes). Block 1's parse hashes positions +/// 0..=4 (each has 4 bytes of forward context); positions 5/6/7 are the +/// unhashable tail. /// - Block 2 = `b"BCDBCDBCDB"` (10 bytes). At absolute position 8 /// (block 2 start) the 4-byte window is `b"BCDB"`. The ONLY place /// `b"BCDB"` was inserted in the hash + chain tables is position 5 -/// — via `backfill_boundary_positions` on the next-slice entry -/// (the 4-byte window at position 5 is `data[5..9] = b"BCD" + -/// block_2[0] = b"BCDB"`). +/// — via the backfill on the next-slice entry (the 4-byte window at +/// position 5 is `data[5..9] = b"BCD" + block_2[0] = b"BCDB"`). /// -/// If `backfill_boundary_positions` regresses, position 5 is never -/// hashed, position 8's lookup misses, and the lazy parser falls -/// through to a leading literals run — `offset == 3, match_len >= 4` -/// would no longer hold. +/// If the backfill regresses, position 5 is never hashed, position 8's +/// lookup misses, and the lazy parser falls through to a leading literals +/// run — `offset == 3, match_len >= 4` would no longer hold. #[test] fn hashchain_inserts_tail_positions_for_next_block_matching() { - let mut matcher = HcMatchGenerator::new(1 << 22); - matcher.configure(HC_CONFIG, super::super::strategy::StrategyTag::Lazy, 22); + // A window of 2^14 searches the hash chain. + let mut matcher = RowMatchGenerator::new(1 << 14); + matcher.configure(ROW_CONFIG); + assert!(matcher.uses_hash_chain()); - matcher.table.commit_input(b"PQRSTBCD"); + matcher.commit_input(b"PQRSTBCD"); let mut history = alloc::vec::Vec::new(); let mut replay = BlockReplay::new(b"PQRSTBCD"); - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { assert!( matches!(seq, Sequence::Literals { .. }), "first block has no internal repeats" ); replay.apply(&mut history, seq); - }); + }))); assert_eq!(history, b"PQRSTBCD"); - matcher.table.commit_input(b"BCDBCDBCDB"); + matcher.commit_input(b"BCDBCDBCDB"); let mut first_sequence_offset: Option = None; let mut first_sequence_match_len: Option = None; - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if first_sequence_offset.is_some() { return; } @@ -4783,7 +4695,7 @@ fn hashchain_inserts_tail_positions_for_next_block_matching() { Sequence::Literals { .. } => { panic!( "expected tail-anchored cross-block match before any literals — \ - backfill_boundary_positions did not seed positions 5/6/7" + the boundary backfill did not seed positions 5/6/7" ) } Sequence::Triple { @@ -4796,16 +4708,15 @@ fn hashchain_inserts_tail_positions_for_next_block_matching() { first_sequence_match_len = Some(match_len); } } - }); + }))); - let offset = first_sequence_offset.expect( - "expected tail-anchored cross-block match emitted from backfill_boundary_positions", - ); + let offset = first_sequence_offset + .expect("expected tail-anchored cross-block match emitted from the boundary backfill"); assert!( (1..=3).contains(&offset), "boundary match offset {offset} must point into the unhashable tail \ (positions 5/6/7 of an 8-byte block 1) so the test specifically \ - locks down backfill_boundary_positions", + locks down the boundary backfill", ); assert_eq!( offset, 3, @@ -4833,7 +4744,7 @@ fn dfast_dense_skip_matching_backfills_previous_tail_for_next_block() { matcher.commit_input(&second); let mut first_sequence = None; - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if first_sequence.is_some() { return; } @@ -4845,7 +4756,7 @@ fn dfast_dense_skip_matching_backfills_previous_tail_for_next_block() { match_len, } => (literal_len, offset, match_len), }); - }); + }))); let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); assert_eq!( @@ -4879,7 +4790,7 @@ fn dfast_sparse_skip_matching_preserves_tail_cross_block_match() { matcher.commit_input(&second); let mut first_sequence = None; - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if first_sequence.is_some() { return; } @@ -4891,7 +4802,7 @@ fn dfast_sparse_skip_matching_preserves_tail_cross_block_match() { match_len, } => (literal_len, offset, match_len), }); - }); + }))); let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); assert_eq!( @@ -5104,7 +5015,7 @@ fn dfast_sparse_skip_matching_backfills_previous_tail_for_consecutive_sparse_blo matcher.commit_input(&third); let mut first_sequence = None; - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if first_sequence.is_some() { return; } @@ -5116,7 +5027,7 @@ fn dfast_sparse_skip_matching_backfills_previous_tail_for_consecutive_sparse_blo match_len, } => (literal_len, offset, match_len), }); - }); + }))); let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); assert_eq!( diff --git a/zstd/src/encoding/match_table/helpers.rs b/zstd/src/encoding/match_table/helpers.rs index 258686a1c..ad36dc662 100644 --- a/zstd/src/encoding/match_table/helpers.rs +++ b/zstd/src/encoding/match_table/helpers.rs @@ -15,7 +15,7 @@ //! change is `pub(crate)` visibility on the moved items. use super::super::opt::types::MatchCandidate; -use crate::encoding::fastpath::FastpathKernel; +use crate::encoding::fastpath::EncodeTier; /// Minimum match length emitted by the Simple / level-1 matcher (and the /// upstream zstd `ZSTD_fast.c` baseline). Lives here so every matcher and the @@ -67,36 +67,15 @@ pub(crate) const INCOMPRESSIBLE_SKIP_STEP: usize = crate::encoding::incompressible::RAW_SKIP_INDEX_STEP; /// Length of the common prefix of two byte slices, capped at -/// `min(a.len(), b.len())`. Hot path on every match finder; dispatches to -/// the per-CPU `common_prefix_len_ptr` kernel (NEON / SSE4.2 / AVX2+BMI2 / -/// scalar) so the SIMD/CRC heavy lifting lives in one place. See -/// [`crate::encoding::fastpath`] for the per-CPU implementations. +/// `min(a.len(), b.len())`, with the compare kernel of the encoder tier `K`. +/// See [`crate::encoding::fastpath`] for the per-tier implementations. #[inline(always)] -pub(crate) fn common_prefix_len(a: &[u8], b: &[u8]) -> usize { +pub(crate) fn common_prefix_len(a: &[u8], b: &[u8]) -> usize { let max = a.len().min(b.len()); // SAFETY: slice `a` / `b` guarantee at least their `len()` initialized // bytes; `max` is the minimum so both pointers are valid for `max` - // bytes. - unsafe { - crate::encoding::fastpath::dispatch_common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) - } -} - -/// As [`common_prefix_len`] but against an already-resolved [`FastpathKernel`]. -/// Hot match-finder loops resolve the kernel once per block and pass it in, -/// so each byte-compare skips the per-call `select_kernel()` `OnceLock` atomic. -#[inline(always)] -pub(crate) fn common_prefix_len_with_kernel(kernel: FastpathKernel, a: &[u8], b: &[u8]) -> usize { - let max = a.len().min(b.len()); - // SAFETY: as `common_prefix_len` — both pointers are valid for `max` bytes. - unsafe { - crate::encoding::fastpath::dispatch_common_prefix_len_ptr_with_kernel( - kernel, - a.as_ptr(), - b.as_ptr(), - max, - ) - } + // bytes. The tier is the one the encoder chose for the running CPU. + unsafe { K::common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) } } /// Pick the better of two candidate matches: longer wins, ties go to @@ -166,8 +145,7 @@ pub(crate) fn extend_backwards_shared( /// invoked once per encoded byte on lazy / Dfast / Row matchers /// (~10% exclusive on the default-level profile). #[inline(always)] -pub(crate) fn repcode_candidate_shared( - kernel: FastpathKernel, +pub(crate) fn repcode_candidate_shared( concat: &[u8], history_abs_start: usize, offset_hist: [u32; 3], @@ -233,8 +211,7 @@ pub(crate) fn repcode_candidate_shared( == core::ptr::read_unaligned(base.add(current_idx).cast::()) }; if gate { - let match_len = common_prefix_len_with_kernel( - kernel, + let match_len = common_prefix_len::( &concat[candidate_idx..], &concat[current_idx..], ); diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index 2753866c3..cd32269c8 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -1,19 +1,9 @@ -//! Shared match-finder storage. +//! Match-finder storage of the optimal parsers. //! -//! `MatchTable` owns every byte of state that both the hash-chain (HC / -//! lazy / lazy2) and binary-tree (BT / optimal parser) backends touch: -//! the rolling window, the contiguous `history` mirror, the absolute -//! position cursors, the hash / hash3 / chain (or BT pointer-pair) -//! tables, and the dictionary-priming flags. Both backends operate on -//! the same physical buffers; the only difference is the semantics of -//! `chain_table` entries — HC mode threads single-link chain pointers -//! through it, BT mode lays out pairs of pointers per node — and that -//! interpretation is the matcher's concern, not the table's. -//! -//! Extracted from `HcMatchGenerator` in #111 Phase 1d Stage 1 so the -//! follow-up stages can pull the HC and BT matchers into their own -//! modules without dragging this shared storage around as a forest of -//! `&mut Vec` arguments. +//! `MatchTable` owns every byte of state the binary-tree search touches: the +//! rolling window, the contiguous `history` mirror, the absolute position +//! cursors, the hash / hash3 / binary-tree pointer-pair tables, and the +//! dictionary-priming flags. use alloc::collections::VecDeque; use alloc::vec::Vec; @@ -22,8 +12,9 @@ use super::super::Sequence; use super::super::blocks::encode_offset_with_history; use super::super::cost_model::HC_OPT_NUM; use super::super::dict_attach::DictAttach; +use super::super::fastpath::EncodeTier; use super::super::hc::HC_MIN_MATCH_LEN; -use super::super::opt::types::{HcOptimalSequence, MatchCandidate}; +use super::super::opt::types::HcOptimalSequence; use super::super::workspace::{HistoryBuf, Table, Workspace, region_bytes}; use super::helpers::INCOMPRESSIBLE_SKIP_STEP; @@ -175,33 +166,24 @@ pub(crate) struct BtCoords { pub(crate) bt_mask: usize, } -/// Shared storage backing every match finder. Holds the contiguous -/// Immutable dictionary match structure (upstream zstd `ZSTD_dictMatchState`) for -/// the binary-tree / optimal path. A hash + single-link chain over the -/// dictionary content concat range, searched in addition to the live BT -/// with its OWN compare budget so dictionary candidates are reachable even -/// when the live tree's budget is spent on recent positions. Slots store -/// `dict_rel + 1` (dictionary-relative concat index); `0` is empty. -/// Which matcher built a [`DmsDictTables`]: the hash-chain (`prime_dms_hc`) and -/// the binary-tree (`prime_dms_bt`) populate `hash_table` / `chain_table` with -/// incompatible meanings, so a cached dms must only be reused by the SAME -/// builder. Part of the cache-reuse key alongside `mls` / `hash_log` / region so -/// a reused compressor that switches level across the HC↔BT boundary (same -/// `mls` / `hash_log`) rebuilds instead of reinterpreting the other layout. +/// Whether a [`DmsDictTables`] holds a built tree; part of the cache-reuse key +/// alongside `mls` / `hash_log` / region. #[derive(Clone, Copy, PartialEq, Eq, Default, Debug)] pub(crate) enum DmsDictLayout { /// Unbuilt / invalidated — never matches a reuse probe. #[default] None, - /// Hash-chain dms (`prime_dms_hc`): single-link chain for the lazy backend. - Hc, /// Binary-tree dms (`prime_dms_bt`): unsorted DUBT for the optimal backend. Bt, } +/// Immutable dictionary match structure (upstream zstd `ZSTD_dictMatchState`) +/// of the optimal path: a binary tree over the dictionary content, searched in +/// addition to the live tree. Slots store `dict_rel + 1` (dictionary-relative +/// concat index); `0` is empty. #[derive(Clone, Default)] pub(crate) struct DmsDictTables { - /// Builder that populated the tables (HC vs BT), part of the reuse key. + /// Whether the tables are built, part of the reuse key. pub(crate) layout: DmsDictLayout, /// Per-hash-bucket binary-tree root (`1 << hash_log` entries), packed /// `dict_rel + 1` (`0` = empty). @@ -219,10 +201,8 @@ pub(crate) struct DmsDictTables { pub(crate) mls: usize, } -/// history buffer, the rolling window, and the hash / chain / hash3 -/// tables. Methods on this struct contain only logic that's identical -/// between HC and BT modes — backend-specific table interpretation -/// lives in the matcher modules. +/// Storage of the optimal parsers' binary-tree search: the contiguous history +/// buffer, the rolling window, and the hash / tree / hash3 tables. pub(crate) struct MatchTable { pub(crate) max_window_size: usize, /// Per-chunk lengths of the live window, in add order. The bytes @@ -290,17 +270,11 @@ pub(crate) struct MatchTable { /// `configure()` so the BT walker and rebase machinery can stay /// on `MatchTable` without consulting the outer generator. pub(crate) is_btultra2: bool, - /// Whether the active backend is one of the BT parsers (`btopt`, - /// `btultra`, `btultra2`). Mirrored from the active strategy for - /// the same reason as `is_btultra2`. - pub(crate) uses_bt: bool, /// Binary-tree finder hash width (`mls`), upstream zstd `ZSTD_hashPtr` mls = - /// `BOUNDED(4, cParams.minMatch, 6)`. clevels.h gives the btlazy2 / btopt - /// band (levels 13-16) `minMatch = 5` → a 5-byte (8-byte read) hash that - /// shortens the chains the BT walks (speed), so it is set per level in - /// `configure`. Only the BT body reads it (the BT hash sites already - /// guarantee 8 readable bytes); the HC `hash_position` stays 4-byte. - /// Defaults to `4`. + /// `BOUNDED(4, cParams.minMatch, 6)`. clevels.h gives btopt level 16 + /// `minMatch = 5` → a 5-byte (8-byte read) hash that shortens the chains + /// the BT walks (speed), so it is set per level in `configure`. The BT hash + /// sites guarantee 8 readable bytes. Defaults to `4`. pub(crate) search_mls: usize, /// Tree coordinates of the block being parsed, taken by /// [`Self::capture_block_coords`] before the parse. @@ -314,22 +288,6 @@ pub(crate) struct MatchTable { /// searched by the BT/optimal collect alongside the live tree. `Some` /// once primed from a non-empty dictionary on a BT level. pub(crate) dms: DictAttach, - /// Borrowed (no-copy) one-shot input window `(ptr, len)`. When set, the - /// scan reads candidate/cursor bytes straight from the caller's slice - /// instead of the owned `history` mirror, so an over-window one-shot - /// input is matched in place (no input->mirror memmove). Raw pointer: - /// live until `clear_borrowed_window` / `reset`. - pub(crate) borrowed_input: Option<(*const u8, usize)>, - /// Active borrowed block range `[start, end)` within `borrowed_input`, - /// staged before each borrowed scan. - pub(crate) borrowed_block: Option<(usize, usize)>, - /// SIMD kernel tier, resolved ONCE at construction (upstream-style - /// once-per-encoder-invoke detection: the CPU does not change mid-run, so - /// `select_kernel()`'s CPUID/`OnceLock` read is paid a single time here). - /// The per-block strategy entries and the BT walker dispatch off this - /// cached value instead of re-detecting in the hot path. Mirrors the - /// `kernel` field the Fast / Row / Dfast matchers already cache. - pub(crate) kernel: crate::encoding::fastpath::FastpathKernel, /// Set by [`Self::retire_history`] ahead of a layout and taken by the next /// [`Self::reset`], which would otherwise read the floor off a history /// already cut down to what it keeps. @@ -370,16 +328,12 @@ impl Clone for MatchTable { dict_resident: self.dict_resident, search_depth: self.search_depth, is_btultra2: self.is_btultra2, - uses_bt: self.uses_bt, search_mls: self.search_mls, block_coords: self.block_coords, // Points into this table's history, not the copy's; taken again // when the copy parses a block. block_live: (core::ptr::null(), 0), dms: self.dms.clone(), - borrowed_input: self.borrowed_input, - borrowed_block: self.borrowed_block, - kernel: self.kernel, retired: self.retired, } } @@ -418,14 +372,10 @@ impl Clone for MatchTable { self.dict_resident = source.dict_resident; self.search_depth = source.search_depth; self.is_btultra2 = source.is_btultra2; - self.uses_bt = source.uses_bt; self.search_mls = source.search_mls; self.block_coords = source.block_coords; // The source's view points into the source's history. self.block_live = (core::ptr::null(), 0); - self.borrowed_input = source.borrowed_input; - self.borrowed_block = source.borrowed_block; - self.kernel = source.kernel; self.retired = source.retired; } } @@ -447,16 +397,14 @@ pub(crate) fn dms_hash_log(region: usize, hash_log: usize) -> usize { (usize::BITS - (region - 1).leading_zeros()).clamp(10.min(hash_log), hash_log) as usize } -/// Shared scaffold for the two `prime_dms_*` dictionary-match-state builders. +/// Scaffold of the dictionary-match-state builder. /// -/// Both resolve the dict `region` (capped to the live history), bail on an -/// undersized dict, derive the dict-sized hash log, short-circuit when a -/// same-shape table is already primed, recycle the prior tables' capacity, run -/// the layout-specific `$build` over the dict bytes at the front of the live -/// history, then publish the tables and mark the cache primed. Only the build -/// pass (HC single-link chain vs BT binary tree), the minimum readable region, -/// and the chain entries per dict position differ — those are the macro -/// parameters. `$build` sees `concat` (the dict bytes), `&mut` `hash_table` / +/// Resolves the dict `region` (capped to the live history), bails on an +/// undersized dict, derives the dict-sized hash log, short-circuits when a +/// same-shape table is already primed, recycles the prior tables' capacity, +/// runs `$build` over the dict bytes at the front of the live history, then +/// publishes the tables and marks the cache primed. `$build` sees `concat` +/// (the dict bytes), `&mut` `hash_table` / /// `chain_table`, `dms_hash_log`, and `region` in scope; `mls` and any /// build-only widths come from the caller's surrounding locals. macro_rules! build_dms { @@ -553,14 +501,10 @@ impl MatchTable { dict_resident: false, search_depth: 0, is_btultra2: false, - uses_bt: false, search_mls: 4, block_coords: BtCoords::default(), block_live: (core::ptr::null(), 0), dms: DictAttach::new(), - borrowed_input: None, - borrowed_block: None, - kernel: crate::encoding::fastpath::select_kernel(), retired: None, } } @@ -655,15 +599,8 @@ impl MatchTable { return; } let idx = abs_pos - self.history_abs_start; - // Borrowed-aware: hash the SAME bytes the finder reads via - // `live_history()` (borrowed window in no-copy mode, owned mirror - // otherwise). Reading `self.history[history_start..]` directly hashed - // the EMPTY owned mirror on the borrowed one-shot path, so every hash3 - // insert early-returned and the HC3 side table stayed empty — btultra2 - // short-match probing then found nothing. This is the identical bug - // already fixed for `insert_position_no_rebase`; the hash3 inserter was - // missed. `hash3` is the only value needed past the borrow, so it ends - // before the table write. + // `hash3` is the only value needed past the borrow, so it ends before + // the table write. let hash3 = { let concat = self.live_history(); if idx + 4 > concat.len() { @@ -677,60 +614,12 @@ impl MatchTable { self.hash3_table_mut()[hash3] = relative_pos + 1; } - /// Insert a position into the main hash / chain table without - /// running the rebase check. Caller pre-validates that the - /// position is representable as a `(rel + 1)` u32, either via - /// `maybe_rebase_positions` (HC) or `bt_update_tree_until` (BT). - /// Upstream zstd parity: `ZSTD_insertAndFindFirstIndex` inner body. - #[inline] - pub(crate) fn insert_position_no_rebase(&mut self, abs_pos: usize) { - let idx = abs_pos.wrapping_sub(self.history_abs_start); - // Borrowed-aware: read the SAME bytes the finder hashes via - // `live_history()` (borrowed window in no-copy mode, owned mirror - // otherwise). Reading `self.history[history_start..]` directly would - // hash the empty owned mirror on the borrowed HC lazy/greedy path - // (live=input, mirror=0) — the insert would skip or hash garbage while - // the finder hashes the real input, so the chain stays empty and no - // matches are ever found. `hash` is the only value needed past this - // borrow, so it ends before the table mutation below. - let hash = { - let concat = self.live_history(); - if idx + 4 > concat.len() { - return; - } - Self::hash_position_at(concat, idx, self.hash_log, 4) - }; - let Some(relative_pos) = self.relative_position(abs_pos) else { - return; - }; - let stored = relative_pos + 1; - let chain_mask = (1usize << self.chain_log) - 1; - let chain_idx = relative_pos as usize & chain_mask; - // SAFETY: `hash` is produced by `hash_value_with_mls` which masks - // the result down to `hash_log` bits, and `hash_table.len() == 1 << - // hash_log` (`ensure_tables`). `chain_idx` is `& chain_mask` so - // `< chain_table.len() == 1 << chain_log`. Both indices are - // provably in bounds, so the elided bounds checks save ~4 - // instructions per call on this per-byte-of-input hot path. - debug_assert!(hash < self.chain_off); - debug_assert!(chain_idx < self.hash3_off - self.chain_off); - // Both regions come from the one buffer; the split ends with this - // statement, so it costs the same two base pointers the separate - // vectors did. - let (hash_tbl, chain_tbl) = self.hash_and_chain_mut(); - unsafe { - let prev = *hash_tbl.get_unchecked(hash); - *chain_tbl.get_unchecked_mut(chain_idx) = prev; - *hash_tbl.get_unchecked_mut(hash) = stored; - } - } - - /// Allocate the hash / chain / hash3 tables sized to the current + /// Allocate the hash / tree / hash3 tables sized to the current /// `hash_log` / `chain_log` / `hash3_log` configuration. Each table /// is reallocated whenever its current length doesn't match the - /// width implied by its `*_log` field — `insert_position_no_rebase` - /// reaches via `get_unchecked`, so a stale-width table after a - /// level change would index out of bounds (UB) on the next encode. + /// width implied by its `*_log` field — the tree walk reaches them + /// through raw pointers, so a stale-width table after a level change + /// would index out of bounds (UB) on the next encode. pub(crate) fn ensure_tables(&mut self) { let (hash_size, chain_size, total) = self.table_sizes(); // The three regions share one buffer, so a width change on any of them @@ -930,13 +819,6 @@ impl MatchTable { Self::hash_value_with_mls(value, hash_log, mls) } - /// Main hash for the current matcher (4-byte MLS, `hash_log` from - /// the table's configuration). - #[inline(always)] - pub(crate) fn hash_position(&self, data: &[u8]) -> usize { - Self::hash_position_with_mls(data, self.hash_log, 4) - } - /// 3-byte hash used by the HC3 side table. Test-only — the /// production path uses inlined per-kernel variants. #[cfg(test)] @@ -976,8 +858,8 @@ impl MatchTable { /// position (`[smaller_child, larger_child]`), `hash_table` the per-bucket /// tree roots. `dict_rel + 1` packing, `0` sentinel. Dict positions are /// concat indices at the front of the shared buffer, so the walk's offset is - /// `idx - dict_idx` directly (no upstream zstd `dmsIndexDelta`). Called on BT levels - /// (`uses_bt`); cached across frames via `DictAttach::is_primed`. + /// `idx - dict_idx` directly (no upstream zstd `dmsIndexDelta`). Cached across + /// frames via `DictAttach::is_primed`. pub(crate) fn prime_dms_bt(&mut self, region_len: usize) { // Upstream zstd `ZSTD_dictMatchState` searches the dict at the live cParams // `minMatch` (= our `search_mls`), NOT a fixed 3 — a fixed 3 surfaces @@ -1053,44 +935,6 @@ impl MatchTable { ); } - /// Build a SEPARATE dictionary match state for the lazy hash-chain path - /// (upstream `ZSTD_dictMatchState`, the dms HC4 walk in - /// `ZSTD_HcFindBestMatch` `zstd_lazy.c:748-770`) over the first `region_len` - /// bytes of the live history (the dictionary at the front). Unlike - /// [`Self::prime_dms_bt`] this is a single-link hash chain (one `next` entry - /// per dict position, like the live HC chain), not a binary tree: the lazy - /// parser walks it with a bounded compare budget rather than descending to - /// the longest match. `hash_table` holds per-bucket chain heads, packed - /// `dict_rel + 1` (`0` = empty); `chain_table[dict_rel]` is the previous - /// dict position in the same bucket, same packing. Dict positions are concat - /// indices at the front of the shared buffer, so the offset is `idx - - /// dict_idx` directly. Hashed at `HC_MIN_MATCH_LEN` (4), matching the live - /// chain. Cached across frames via `DictAttach::is_primed`. - pub(crate) fn prime_dms_hc(&mut self, region_len: usize) { - let mls = HC_MIN_MATCH_LEN; - let floor = self.dict_index_floor; - // Single `next` link per dict position (HC4 chain, not the BT's 2/pos). - build_dms!( - self, - region_len, - DmsDictLayout::Hc, - mls, - mls, - 1, - |concat, hash_table, chain_table, dms_hash_log, region| { - let mut current = floor; - while current + mls <= region { - let h = Self::hash_position_at(concat, current, dms_hash_log, mls); - // Prepend `current` to its bucket chain (upstream zstd - // `ZSTD_insertAndFindFirstIndex` head insert). - chain_table[current] = hash_table[h]; - hash_table[h] = (current + 1) as u32; - current += 1; - } - } - ); - } - /// Heap bytes this table owns: history, the hash / hash3 / chain tables, /// the chunk-length deque, and any attached immutable dictionary tables. pub(crate) fn heap_size(&self) -> usize { @@ -1246,21 +1090,6 @@ impl MatchTable { /// `history` Vec. #[inline] pub(crate) fn live_history(&self) -> &[u8] { - // Borrowed one-shot: expose `[0, block_end)` of the caller's input so - // candidate/cursor reads land in place (the owned `history` mirror is - // empty under the no-copy path). Loop-invariant branch, inlines. - if let Some((_start, end)) = self.borrowed_block { - let (ptr, total) = self - .borrowed_input - .expect("borrowed_block set without a registered borrowed window"); - debug_assert!( - end <= total, - "borrowed block end {end} exceeds window {total}" - ); - // SAFETY: `ptr` is the registered window start (live by contract); - // `end <= total` in bounds. - return unsafe { core::slice::from_raw_parts(ptr, end) }; - } // Stop at the committed end, not at `history.len()`: the next block's // bytes may already be read into the tail, and a scan must not see // past the block it is compressing. @@ -1276,65 +1105,20 @@ impl MatchTable { /// the most recent buffer in the rolling window — panics if no /// data has been committed yet. pub(crate) fn get_last_space(&self) -> &[u8] { - if let (Some((ptr, _total)), Some((block_start, block_end))) = - (self.borrowed_input, self.borrowed_block) - { - // SAFETY: borrowed liveness contract; range validated when staged. - return unsafe { - core::slice::from_raw_parts(ptr.add(block_start), block_end - block_start) - }; - } let last = *self.chunk_lens.back().unwrap(); &self.history[self.history.len() - self.uncommitted_len - last ..self.history.len() - self.uncommitted_len] } - /// Register the borrowed input window. Zeroes `history_abs_start` so - /// borrowed positions are absolute input offsets (the owned floor-advance - /// reset leaves it non-zero across frames; a borrowed frame commits - /// nothing, so it stays 0 for the frame). Every probed candidate is byte-verified, so - /// stale table entries from a prior frame are rejected (or, if their bytes - /// coincidentally match, are genuine in-window matches). - /// - /// # Safety - /// `buffer` must stay live and unmodified until `clear_borrowed_window` - /// or `reset`. - pub(crate) unsafe fn set_borrowed_window(&mut self, buffer: &[u8]) { - self.borrowed_input = Some((buffer.as_ptr(), buffer.len())); - self.borrowed_block = None; - self.history_abs_start = 0; - } - - pub(crate) fn clear_borrowed_window(&mut self) { - self.borrowed_input = None; - self.borrowed_block = None; - } - - /// Stage `[block_start, block_end)` as the active borrowed block. - pub(crate) fn stage_borrowed_block(&mut self, block_start: usize, block_end: usize) { - let (_ptr, total) = self - .borrowed_input - .expect("stage_borrowed_block requires a registered borrowed window"); - assert!( - block_start <= block_end && block_end <= total, - "borrowed block bounds out of range: start={block_start} end={block_end} total={total}", - ); - self.borrowed_block = Some((block_start, block_end)); - } - - /// `(current_abs_start, current_len)` for the active scan. Borrowed: the - /// staged block range. Owned: the last committed chunk in the live window, - /// or an empty range at the window's end when none has been committed. + /// `(current_abs_start, current_len)` for the active scan: the last + /// committed chunk in the live window, or an empty range at the window's + /// end when none has been committed. pub(crate) fn current_block_range(&self) -> (usize, usize) { - if let Some((start, end)) = self.borrowed_block { - (start, end - start) - } else { - let current_len = self.chunk_lens.back().copied().unwrap_or(0); - ( - self.history_abs_start + self.window_size - current_len, - current_len, - ) - } + let current_len = self.chunk_lens.back().copied().unwrap_or(0); + ( + self.history_abs_start + self.window_size - current_len, + current_len, + ) } /// Test-only: append a chunk to the live window without eviction or @@ -1596,10 +1380,6 @@ impl MatchTable { self.window_size = 0; self.offset_hist = [1, 4, 8]; self.skip_insert_until_abs = 0; - // Clear borrowed-window state so a following OWNED frame reads the - // owned mirror; a borrowed frame re-arms via `set_borrowed_window`. - self.borrowed_input = None; - self.borrowed_block = None; self.dictionary_primed_for_frame = false; if let Some(region) = reborrow_region { // Keep `[0, region)` (the dict); drop the previous frame's input. @@ -1743,7 +1523,7 @@ impl MatchTable { /// `target_feature` umbrella. Every position in the range must already be /// representable (see [`Self::arm_block_positions`]). #[inline(always)] - pub(crate) fn bt_insert_range( + pub(crate) fn bt_insert_range( &mut self, from: usize, stop: usize, @@ -1761,7 +1541,7 @@ impl MatchTable { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match self.kernel { + crate::encoding::fastpath::tier_match!(K { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { self.bt_insert_range_avx2_bmi2(from, stop, current_abs_end, target_abs) @@ -1772,11 +1552,11 @@ impl MatchTable { }, FastpathKernel::Scalar => { self.bt_insert_range_scalar(from, stop, current_abs_end, target_abs) - } - } + }, + }) } // wasm has no runtime CPU detection: `simd128` is resolved at compile - // time, so the tier is picked by `cfg` rather than by `self.kernel`. + // time, so the tier is picked by `cfg` rather than by `K`. #[cfg(all( target_arch = "wasm32", target_feature = "simd128", @@ -1950,9 +1730,13 @@ impl MatchTable { /// BT-side history replay after [`Self::begin_rebase`]. Re-walks /// `history_start..abs_pos` through the tree insertion so the pointer-pair /// table is consistent with the freshly reset `position_base`. - pub(crate) fn replay_history_for_rebase_bt(&mut self, history_start: usize, abs_pos: usize) { + pub(crate) fn replay_history_for_rebase_bt( + &mut self, + history_start: usize, + abs_pos: usize, + ) { let rebuild_end = self.history_abs_end(); - let _ = self.bt_insert_range(history_start, abs_pos, rebuild_end, abs_pos); + let _ = self.bt_insert_range::(history_start, abs_pos, rebuild_end, abs_pos); } /// Insert every position from the insertion frontier up to `abs_pos` @@ -1964,7 +1748,11 @@ impl MatchTable { /// covered stay out of the tree. Clamping to `abs_pos` inserted them and /// gave the tree candidates upstream never surfaces (a longer but farther /// match the optimal parser then forced over a cheaper closer one). - pub(crate) fn bt_update_tree_until(&mut self, abs_pos: usize, current_abs_end: usize) { + pub(crate) fn bt_update_tree_until( + &mut self, + abs_pos: usize, + current_abs_end: usize, + ) { if self.skip_insert_until_abs < self.history_abs_start { self.skip_insert_until_abs = self.history_abs_start; } @@ -1972,7 +1760,7 @@ impl MatchTable { if update_abs < abs_pos { if self.can_skip_rebase_check(abs_pos) { // Every position up to the target is representable: one run. - let _ = self.bt_insert_range(update_abs, abs_pos, current_abs_end, abs_pos); + let _ = self.bt_insert_range::(update_abs, abs_pos, current_abs_end, abs_pos); } else { // A rebase may be due inside the run, and it moves the // coordinates the range hoists, so go one position at a time. @@ -1980,8 +1768,12 @@ impl MatchTable { if !self.can_skip_rebase_check(abs_pos) { self.maybe_rebase_positions(update_abs); } - update_abs = - self.bt_insert_range(update_abs, update_abs + 1, current_abs_end, abs_pos); + update_abs = self.bt_insert_range::( + update_abs, + update_abs + 1, + current_abs_end, + abs_pos, + ); } } } @@ -2135,11 +1927,9 @@ impl MatchTable { } } - /// Cold rebase: clear the hash / hash3 / chain tables and replay - /// the inserted history prefix through the active backend's walker - /// so the new `position_base` is consistent. The `uses_bt` flag - /// (mirrored from `HcParseMode`) selects between the HC and BT - /// replay variants. + /// Cold rebase: clear the hash / hash3 / tree tables and replay the + /// inserted history prefix through the tree walk so the new + /// `position_base` is consistent. #[cold] #[inline(never)] pub(crate) fn rebase_positions_cold(&mut self, abs_pos: usize) { @@ -2147,11 +1937,15 @@ impl MatchTable { let history_start = self.history_abs_start; // Rebuild only the already-inserted prefix. The caller inserts abs_pos // immediately after this, and later positions are added in-order. - if self.uses_bt { - self.replay_history_for_rebase_bt(history_start, abs_pos); - } else { - self.replay_history_for_rebase_hc(history_start, abs_pos); - } + // + // With the scalar compare, fixed: this runs once per ~4 GiB of stream, + // below every position insert, and the tiers' compares return the same + // lengths, so the tree is the same and only this rare pass's speed + // depends on it. + self.replay_history_for_rebase_bt::( + history_start, + abs_pos, + ); // begin_rebase() also clears `hash3_table`. Rewind `next_to_update3` // to the inserted prefix start so `update_hash3_until` re-fills the // HC3 side table — without this, every HC3 probe up to `abs_pos` @@ -2163,165 +1957,10 @@ impl MatchTable { } } - /// Insert a single position into the hash / chain tables, rebasing - /// first if required. - #[inline] - pub(crate) fn insert_position(&mut self, abs_pos: usize) { - self.maybe_rebase_positions(abs_pos); - self.insert_position_no_rebase(abs_pos); - } - - /// Insert every position in `[start, end)` into the hash / chain - /// tables and advance the hash3 fill cursor past `end`. - /// - /// The rebase guard is hoisted out of the per-position loop: a single - /// `(rel + 1)`-representability check on both ends of the range decides - /// whether the whole span fits without a rebase. The check is monotone - /// in `pos` (rebase only fires as the relative position approaches - /// `u32::MAX`), so when neither end needs a rebase no interior position - /// can either, and the - /// fill runs as a tight loop with raw `(pos - position_base)` index - /// arithmetic and hoisted base pointers. This mirrors the upstream zstd's - /// once-per-block `ZSTD_window_correctOverflow` followed by an - /// unchecked fill, and matters on highly repetitive inputs where a - /// single long match makes this loop fill the entire block. When a - /// rebase is required the cold per-position path runs unchanged. - pub(crate) fn insert_positions(&mut self, start: usize, end: usize) { - if start < end { - if self.needs_rebase(start) || self.needs_rebase(end - 1) { - // Cold path: at least one position in the range needs a - // rebase. Defer to the guarded per-position insert, which - // rebases exactly when each position requires it. - for pos in start..end { - self.insert_position(pos); - } - } else { - self.fill_hash_chain_positions(start, end); - } - } - self.next_to_update3 = self.next_to_update3.max(end); - } - - /// Index a just-emitted match span with the upstream zstd skip-threshold cap - /// (`ZSTD_row_update_internal`, `zstd_lazy.c:922-940`): when the span - /// exceeds `SKIP_THRESHOLD` positions only the first `MAX_START` and last - /// `MAX_END` are chained, the interior is skipped. Indexing every interior - /// byte of a long match is O(matchlen) and dominates HashChain (lazy) - /// encode time on periodic inputs where one match can span a whole block; - /// upstream zstd's hash-chain finder chains nothing in the interior at all, so the - /// 96 + 32 cap is a conservative (ratio-preserving) mirror that still keeps - /// boundary anchors for the following search. - pub(crate) fn insert_match_span(&mut self, start: usize, end: usize) { - const SKIP_THRESHOLD: usize = 384; - const MAX_START: usize = 96; - const MAX_END: usize = 32; - if end.saturating_sub(start) > SKIP_THRESHOLD { - // Raw arithmetic is correct by design here, NOT masked with - // `saturating_*`. `start` / `end` are absolute stream - // positions, and `check_stream_abs_headroom` guarantees - // `abs_pos + STREAM_ABS_HEADROOM (= 4112) <= usize::MAX` for - // every position in the frame, so `start + MAX_START` (96) - // cannot overflow even on a 32-bit target. In this branch - // `end - start > SKIP_THRESHOLD (384)`, so `end > 384 > - // MAX_END (32)` and `end - MAX_END` cannot underflow. - self.insert_positions(start, start + MAX_START); - self.insert_positions(end - MAX_END, end); - } else { - self.insert_positions(start, end); - } - } - - /// Tight hash/chain fill for `[start, end)` when the caller has already - /// proven every position is `(rel + 1)`-representable (so no rebase can - /// occur). Equivalent to looping - /// [`Self::insert_position_no_rebase`], but with the table base pointers - /// and config hoisted out of the loop and the relative position derived - /// by a raw subtraction instead of the checked `relative_position` - /// arithmetic. Upstream zstd parity: the unchecked `ZSTD_insertAndFindFirstIndex` - /// fill body. - #[inline] - fn fill_hash_chain_positions(&mut self, start: usize, end: usize) { - let history_abs_start = self.history_abs_start; - let position_base = self.position_base; - let index_shift = self.index_shift; - let hash_log = self.hash_log; - let chain_mask = (1usize << self.chain_log) - 1; - // Borrowed-aware source: `live_history()` is the borrowed input window - // in no-copy mode (owned mirror empty) and `history[history_start..]` - // otherwise. Reading `self.history` directly would hash the empty - // mirror on the borrowed HC lazy/greedy path. The raw ptr is copied - // out so it holds no borrow across the `hash_table` / `chain_table` - // mutations below; the window stays valid for the loop (borrowed input - // is live by contract, owned `history` is not realloc'd mid-fill). - let (concat_ptr, concat_len) = { - let live = self.live_history(); - (live.as_ptr(), live.len()) - }; - debug_assert!(self.chain_off == 1usize << hash_log); - debug_assert!(self.hash3_off - self.chain_off == 1usize << self.chain_log); - // Base pointers into the one buffer, hoisted before the loop below. - // SAFETY: `chain_off` is in bounds (one past the hash region is the - // first chain slot), asserted just above. - let hash_ptr = self.tables.as_mut_ptr(); - let chain_ptr = unsafe { hash_ptr.add(self.chain_off) }; - // The last `< 4` bytes of the live window can't be hashed. Compute - // the hashable upper bound once instead of branching per position: a - // position `pos` is hashable iff `pos - history_abs_start + 4 <= - // concat_len`, i.e. `pos < history_abs_start + (concat_len - 3)`. - let hashable_end = end.min(history_abs_start + concat_len.saturating_sub(3)); - if start >= hashable_end { - return; - } - // Hoist the source pointer and relative index out of the loop and - // advance both by one per iteration, mirroring the upstream zstd's - // `ip++ / idx++` fill rather than recomputing them from `pos`. - let mut src = unsafe { concat_ptr.add(start - history_abs_start) }; - // `rel` cannot reach `u32::MAX` because the caller proved - // `!needs_rebase(end - 1)`, and the `wrapping_add` below keeps the - // overflow branch off this per-byte hot loop. The distance from the - // floor comes first so the sum stays in range on a 32-bit word. - let mut rel = (start - position_base + index_shift) as u32; - for _ in start..hashable_end { - // SAFETY: every `src` in `[start, hashable_end)` is at least 4 - // bytes from the end of `history`, so the unaligned 4-byte read is - // in range. `hash` is masked to `hash_log` bits and `chain_idx` to - // `chain_log` bits, both within the table lengths asserted above. - unsafe { - let value = Self::read_le_u32_ptr(src); - let hash = Self::hash_value_with_mls(value, hash_log, 4); - let chain_idx = (rel as usize) & chain_mask; - let prev = *hash_ptr.add(hash); - *chain_ptr.add(chain_idx) = prev; - *hash_ptr.add(hash) = rel + 1; - src = src.add(1); - } - rel = rel.wrapping_add(1); - } - } - - /// Insert every `step`-th position in `[start, end)` — the sparse - /// counterpart to [`Self::insert_positions`]. Skipped positions are - /// *not* advanced through `next_to_update3` (the upstream zstd's behaviour - /// for the "incompressible block" skip path). - pub(crate) fn insert_positions_with_step(&mut self, start: usize, end: usize, step: usize) { - if step == 0 { - return; - } - let mut pos = start; - while pos < end { - self.insert_position(pos); - let next = pos.saturating_add(step); - if next <= pos { - break; - } - pos = next; - } - } - /// Backfill the last `< 4` bytes of the previous slice (which couldn't - /// be hashed at the time because `insert_position` needs 4 bytes of - /// lookahead) before the next slice's matching pass. - pub(crate) fn backfill_boundary_positions( + /// be hashed at the time, a position needs 4 bytes of lookahead) before + /// the next slice's matching pass. + pub(crate) fn backfill_boundary_positions( &mut self, current_abs_start: usize, current_abs_end: usize, @@ -2330,11 +1969,7 @@ impl MatchTable { .saturating_sub(3) .max(self.history_abs_start); if backfill_start < current_abs_start { - if self.uses_bt { - self.bt_update_tree_until(current_abs_start, current_abs_end); - } else { - self.insert_positions(backfill_start, current_abs_start); - } + self.bt_update_tree_until::(current_abs_start, current_abs_end); } } @@ -2342,21 +1977,18 @@ impl MatchTable { /// stretch of `skip_insert_until_abs`. Cap the gap at 384 to bound /// the worst-case BT tree update on the next match search. pub(crate) fn apply_limited_update_after_long_match(&mut self, current_abs_start: usize) { - if !self.uses_bt { - return; - } let gap = current_abs_start.saturating_sub(self.skip_insert_until_abs); if gap > 384 { self.skip_insert_until_abs = current_abs_start - (gap - 384).min(192); } } - /// BT-mode counterpart of the HC sparse skip path. Inserts every + /// Sparse skip of an incompressible block. Inserts every /// `INCOMPRESSIBLE_SKIP_STEP`-th position through the BT walker /// (which threads the binary tree) and then densely inserts the /// final `HC_MIN_MATCH_LEN + INCOMPRESSIBLE_SKIP_STEP` positions so /// the very last hashable starts are present for the next slice. - pub(crate) fn bt_insert_sparse_incompressible_block( + pub(crate) fn bt_insert_sparse_incompressible_block( &mut self, current_abs_start: usize, current_abs_end: usize, @@ -2364,7 +1996,7 @@ impl MatchTable { let mut pos = current_abs_start; while pos < current_abs_end { self.maybe_rebase_positions(pos); - let _ = self.bt_insert_range(pos, pos + 1, current_abs_end, current_abs_end); + let _ = self.bt_insert_range::(pos, pos + 1, current_abs_end, current_abs_end); self.insert_hash3_only_no_rebase(pos); let next = pos.saturating_add(INCOMPRESSIBLE_SKIP_STEP); if next <= pos { @@ -2383,7 +2015,7 @@ impl MatchTable { continue; } self.maybe_rebase_positions(pos); - let _ = self.bt_insert_range(pos, pos + 1, current_abs_end, current_abs_end); + let _ = self.bt_insert_range::(pos, pos + 1, current_abs_end, current_abs_end); self.insert_hash3_only_no_rebase(pos); } @@ -2391,9 +2023,6 @@ impl MatchTable { self.next_to_update3 = self.next_to_update3.max(current_abs_end); } - /// `skip_matching` body — backfill the slice boundary and then walk - /// the current slice in either dense or sparse mode (driven by the - /// `incompressible_hint`). BT and HC modes branch via `uses_bt`. /// Dict-priming for BT / optimal levels (upstream zstd `ZSTD_dictMatchState`): /// the dictionary content stays in `history` (so the dms chain can read it /// and offsets are computed across the dict→input boundary) but is NOT @@ -2439,28 +2068,16 @@ impl MatchTable { self.uncommitted_len += len; } - /// Index the last committed block as the body of a long-distance match on - /// the lazy chain: only its tail, which is what upstream's lazy search - /// catches up on after one (`ZSTD_ldm_limitTableUpdate` moves - /// `nextToUpdate` to it), since the match itself is never searched. + /// Index the last committed block as the body of a long-distance match. + /// The tree gets here only behind a producer, which parses in place of the + /// optimal parser: it resumes near the body's end, as after a long match + /// of its own (`ZSTD_ldm_limitTableUpdate`). #[cfg(feature = "ldm")] pub(crate) fn skip_long_match(&mut self) { self.ensure_tables(); let (start, len) = self.current_block_range(); - let end = start + len; - if self.uses_bt { - // The tree gets here only behind a producer, which parses in place - // of the optimal parser: it resumes near the body's end, as after - // a long match of its own. - self.apply_limited_update_after_long_match(end); - self.next_to_update3 = self.next_to_update3.max(self.skip_insert_until_abs); - return; - } - self.backfill_boundary_positions(start, end); - self.insert_positions( - end - len.min(crate::encoding::ldm::LONG_MATCH_INDEXED_TAIL), - end, - ); + self.apply_limited_update_after_long_match(start + len); + self.next_to_update3 = self.next_to_update3.max(self.skip_insert_until_abs); } /// Carry a long-distance match the driver emitted into the repeat @@ -2474,7 +2091,7 @@ impl MatchTable { ); } - pub(crate) fn skip_matching(&mut self, incompressible_hint: Option) { + pub(crate) fn skip_matching(&mut self, incompressible_hint: Option) { self.ensure_tables(); let (current_abs_start, current_len) = self.current_block_range(); let current_abs_end = current_abs_start + current_len; @@ -2482,218 +2099,12 @@ impl MatchTable { // indexed only from its floor on. let floor = (self.history_abs_start + self.dict_index_floor).min(current_abs_end); let current_abs_start = current_abs_start.max(floor); - self.backfill_boundary_positions(current_abs_start, current_abs_end); - if self.uses_bt { - if incompressible_hint == Some(true) { - self.bt_insert_sparse_incompressible_block(current_abs_start, current_abs_end); - return; - } - self.bt_update_tree_until(current_abs_end, current_abs_end); - return; - } + self.backfill_boundary_positions::(current_abs_start, current_abs_end); if incompressible_hint == Some(true) { - self.insert_positions_with_step( - current_abs_start, - current_abs_end, - INCOMPRESSIBLE_SKIP_STEP, - ); - let dense_tail = HC_MIN_MATCH_LEN + INCOMPRESSIBLE_SKIP_STEP; - let tail_start = current_abs_end - .saturating_sub(dense_tail) - .max(self.history_abs_start); - let tail_start = tail_start.max(current_abs_start); - for pos in tail_start..current_abs_end { - if !(pos - current_abs_start).is_multiple_of(INCOMPRESSIBLE_SKIP_STEP) { - self.insert_position(pos); - } - } - } else { - self.insert_positions(current_abs_start, current_abs_end); - } - } - - /// Stage D: HC3 short-match probe. Cross-platform dispatcher. - /// External / test callers only — the on-encode hot path bypasses - /// this via the per-kernel variant from inside the surrounding - /// `collect_optimal_candidates_initialized_` umbrella. - #[allow(dead_code)] - #[inline(always)] - pub(crate) fn hash3_candidate( - &self, - abs_pos: usize, - current_abs_end: usize, - min_match_len: usize, - ) -> Option { - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - unsafe { - self.hash3_candidate_neon(abs_pos, current_abs_end, min_match_len) - } - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - use crate::encoding::fastpath::FastpathKernel; - match self.kernel { - #[cfg(feature = "kernel-avx2")] - FastpathKernel::Avx2Bmi2 => unsafe { - self.hash3_candidate_avx2_bmi2(abs_pos, current_abs_end, min_match_len) - }, - #[cfg(feature = "kernel-sse")] - FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { - self.hash3_candidate_sse2(abs_pos, current_abs_end, min_match_len) - }, - FastpathKernel::Scalar => { - self.hash3_candidate_scalar(abs_pos, current_abs_end, min_match_len) - } - } - } - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - unsafe { - self.hash3_candidate_simd128(abs_pos, current_abs_end, min_match_len) - } - #[cfg(not(any( - all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ), - all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ), - target_arch = "x86", - target_arch = "x86_64" - )))] - { - self.hash3_candidate_scalar(abs_pos, current_abs_end, min_match_len) + self.bt_insert_sparse_incompressible_block::(current_abs_start, current_abs_end); + return; } - } - - /// WebAssembly `simd128` umbrella HC3 probe. - /// - /// # Safety - /// wasm32 with `simd128` enabled at compile time. Body inlines via macro. - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - #[target_feature(enable = "simd128")] - pub(crate) unsafe fn hash3_candidate_simd128( - &self, - abs_pos: usize, - current_abs_end: usize, - min_match_len: usize, - ) -> Option { - super::super::hc::generator::hash3_candidate_body!( - self, - abs_pos, - current_abs_end, - min_match_len, - crate::encoding::fastpath::simd128::common_prefix_len_ptr, - ) - } - - /// NEON umbrella HC3 probe. - /// - /// # Safety - /// AArch64 with NEON (baseline). Body inlines via macro. - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - #[target_feature(enable = "neon")] - pub(crate) unsafe fn hash3_candidate_neon( - &self, - abs_pos: usize, - current_abs_end: usize, - min_match_len: usize, - ) -> Option { - super::super::hc::generator::hash3_candidate_body!( - self, - abs_pos, - current_abs_end, - min_match_len, - crate::encoding::fastpath::neon::common_prefix_len_ptr, - ) - } - - /// SSE2 umbrella HC3 probe. - /// - /// # Safety - /// x86/x86_64 with SSE2. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" - ))] - #[target_feature(enable = "sse2")] - pub(crate) unsafe fn hash3_candidate_sse2( - &self, - abs_pos: usize, - current_abs_end: usize, - min_match_len: usize, - ) -> Option { - super::super::hc::generator::hash3_candidate_body!( - self, - abs_pos, - current_abs_end, - min_match_len, - crate::encoding::fastpath::sse2::common_prefix_len_ptr, - ) - } - - /// AVX2+BMI2 umbrella HC3 probe. - /// - /// # Safety - /// x86/x86_64 with AVX2 + BMI2. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" - ))] - #[target_feature(enable = "avx2,bmi2")] - pub(crate) unsafe fn hash3_candidate_avx2_bmi2( - &self, - abs_pos: usize, - current_abs_end: usize, - min_match_len: usize, - ) -> Option { - super::super::hc::generator::hash3_candidate_body!( - self, - abs_pos, - current_abs_end, - min_match_len, - crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, - ) - } - - /// Scalar fallback HC3 probe, compiled unless the NEON tier covers this - /// target. - #[cfg(not(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - )))] - pub(crate) fn hash3_candidate_scalar( - &self, - abs_pos: usize, - current_abs_end: usize, - min_match_len: usize, - ) -> Option { - super::super::hc::generator::hash3_candidate_body!( - self, - abs_pos, - current_abs_end, - min_match_len, - crate::encoding::fastpath::scalar::common_prefix_len_ptr, - ) + self.bt_update_tree_until::(current_abs_end, current_abs_end); } /// Reset the rebase-derived bookkeeping (rolling `position_base` / @@ -2709,17 +2120,6 @@ impl MatchTable { self.tables.fill(HC_EMPTY); } - /// HC-side history replay after [`Self::begin_rebase`]. Re-inserts every - /// position from `history_start` (inclusive) to `abs_pos` - /// (exclusive) into the HC chain/hash tables without re-checking - /// the rebase guard — the caller has just rebased, so positions - /// are by construction representable. - pub(crate) fn replay_history_for_rebase_hc(&mut self, history_start: usize, abs_pos: usize) { - for pos in history_start..abs_pos { - self.insert_position_no_rebase(pos); - } - } - /// Upstream zstd parity: replay an optimal-parser plan into the consumer's /// sequence sink over a block of `current_len` bytes, advancing /// `offset_hist` exactly like the upstream zstd block-store walker. diff --git a/zstd/src/encoding/match_table/storage/storage_tests.rs b/zstd/src/encoding/match_table/storage/storage_tests.rs index 30ade717a..cc2099ea0 100644 --- a/zstd/src/encoding/match_table/storage/storage_tests.rs +++ b/zstd/src/encoding/match_table/storage/storage_tests.rs @@ -37,54 +37,15 @@ fn set_dictionary_limit_from_primed_bytes_offsets_from_history_start() { assert_eq!(t.dictionary_limit_abs, Some(140)); } -#[test] -fn dms_cache_rebuilds_across_hc_bt_layout_switch() { - // A reused compressor that changes level across the HC↔BT boundary lands - // back in prime_dms_* with the SAME (region, mls, hash_log) but the OTHER - // builder. Without a layout discriminator in the cache key, the second - // prime would reuse the first builder's tables verbatim (HC single-link - // chain reinterpreted as a BT DUBT, or vice versa) — a silent corruption. - // The cache key must include the layout so the switch forces a rebuild. - let mut t = new_table(64); - // dms_hash_log clamps to [10, hash_log]; keep hash_log above the floor. - t.hash_log = 12; - // Match HC's fixed mls (HC_MIN_MATCH_LEN == 4) so the BT prime resolves - // the SAME (region, mls, hash_log) as the HC prime — then the LAYOUT field - // is the only differing key, which is exactly what this test guards. With - // search_mls != 4 the rebuild would happen via the mls mismatch and the - // test would pass even without the layout discriminator. - t.search_mls = 4; - t.push_test_chunk(vec![7u8; 48]); - t.ensure_tables(); - let region = 48; - - t.prime_dms_hc(region); - assert_eq!(t.dms.table().unwrap().layout, DmsDictLayout::Hc); - // HC chain has one `next` per dict position. - assert_eq!(t.dms.table().unwrap().chain_table.len(), region); - - // Same region/mls/hash_log, but the BT builder must NOT reuse the HC - // tables: it rebuilds to the BT layout (2 children per dict position). - t.prime_dms_bt(region); - assert_eq!(t.dms.table().unwrap().layout, DmsDictLayout::Bt); - assert_eq!(t.dms.table().unwrap().chain_table.len(), 2 * region); - - // And back to HC rebuilds again. - t.prime_dms_hc(region); - assert_eq!(t.dms.table().unwrap().layout, DmsDictLayout::Hc); - assert_eq!(t.dms.table().unwrap().chain_table.len(), region); -} - #[test] fn skip_matching_bt_incompressible_routes_through_sparse_block() { let mut t = new_table(32); t.push_test_chunk(vec![0u8; 32]); t.ensure_tables(); - t.uses_bt = true; t.is_btultra2 = false; t.search_depth = 4; let before_skip_until = t.skip_insert_until_abs; - t.skip_matching(Some(true)); + crate::encoding::fastpath::with_host_tier!(K => t.skip_matching::(Some(true))); // BT + incompressible path must take the // `bt_insert_sparse_incompressible_block` branch and advance // `skip_insert_until_abs` to current_abs_end. @@ -97,11 +58,10 @@ fn skip_matching_bt_dense_routes_through_bt_update_tree() { let mut t = new_table(32); t.push_test_chunk(vec![1u8; 32]); t.ensure_tables(); - t.uses_bt = true; t.is_btultra2 = false; t.search_depth = 4; // `incompressible_hint = None` → dense bt_update_tree_until path - t.skip_matching(None); + crate::encoding::fastpath::with_host_tier!(K => t.skip_matching::(None)); assert_eq!(t.skip_insert_until_abs, t.history_abs_start + t.window_size); } @@ -119,14 +79,13 @@ fn replay_history_for_rebase_bt_walks_inserted_prefix() { t.window_size = 64; t.position_base = 0; t.search_depth = 4; - t.uses_bt = true; t.ensure_tables(); // Replay the first 32 positions; the BT walker writes entries // into the hash table (via `hash_table[hash] = stored`) so the // ground-truth observation is "some hash slots are no longer // HC_EMPTY". assert!(t.hash_table().iter().all(|&v| v == HC_EMPTY)); - t.replay_history_for_rebase_bt(0, 32); + crate::encoding::fastpath::with_host_tier!(K => t.replay_history_for_rebase_bt::(0, 32)); assert!( t.hash_table().iter().any(|&v| v != HC_EMPTY), "BT replay must populate hash table" @@ -333,58 +292,77 @@ fn rebase_positions_cold_rebuilds_hash3_for_btultra2() { ); } -#[test] -fn insert_positions_with_step_zero_step_is_noop() { - let mut t = new_table(32); - t.history = vec![0u8; 32].into(); - t.push_test_chunk(vec![0u8; 32]); +/// A binary-tree table over 64 bytes of period-17 input, every position +/// representable from the origin. +fn bt_table() -> MatchTable { + let mut t = new_table(64); + t.history = (0..64u8) + .map(|i| i % 17) + .collect::>() + .into(); + t.history_start = 0; + t.history_abs_start = 0; + t.window_size = 64; + t.position_base = 0; + t.search_depth = 4; t.ensure_tables(); - let next_to_update3_before = t.next_to_update3; - // step=0 must early-return without touching anything. - t.insert_positions_with_step(0, 16, 0); - assert!(t.hash_table().iter().all(|&v| v == HC_EMPTY)); - assert_eq!(t.next_to_update3, next_to_update3_before); + t } +/// The tree the rebase replays is the tree the walk had built: the replay +/// runs on the scalar compare whatever tier built the original, so the two +/// agree only if every tier reports the same lengths, and a rebase that lost +/// or added a node would change what the optimal parser sees afterwards. #[test] -fn insert_positions_with_step_saturating_step_breaks_loop() { - // step = usize::MAX so first iteration overflows - // `pos.saturating_add(step)` to usize::MAX, then the `next <= pos` - // guard breaks out of the loop after one insert. - let mut t = new_table(32); - t.history = vec![1u8; 32].into(); - t.push_test_chunk(vec![1u8; 32]); - t.ensure_tables(); - t.insert_positions_with_step(0, 16, usize::MAX); - // Exactly one position should have been inserted before the - // loop terminated — observe that only one slot is non-empty. - let non_empty = t.hash_table().iter().filter(|&&v| v != HC_EMPTY).count(); +fn rebase_positions_cold_rebuilds_the_binary_tree_the_walk_built() { + let mut t = bt_table(); + crate::encoding::fastpath::with_host_tier!(K => t.bt_update_tree_until::(32, 64)); + let walked = (t.hash_table().to_vec(), t.chain_table().to_vec()); assert!( - non_empty <= 1, - "step=usize::MAX must break after the first insert" + walked.0.iter().any(|&v| v != HC_EMPTY), + "fixture precondition: the walk must populate the tree", + ); + + t.rebase_positions_cold(32); + + assert_eq!(t.position_base, t.history_abs_start); + assert_eq!( + (t.hash_table().to_vec(), t.chain_table().to_vec()), + walked, + "the replayed tree differs from the walked one", ); } +/// A tree walk whose target is past the index range goes one position at a +/// time, rebasing before the first insert, and ends with the tree a walk from +/// a representable start builds. #[test] -fn apply_limited_update_after_long_match_hc_mode_is_noop() { - // HC mode (`uses_bt = false`) — function must early-return - // without mutating `skip_insert_until_abs`. - let mut t = new_table(32); - t.uses_bt = false; - t.skip_insert_until_abs = 100; - t.apply_limited_update_after_long_match(1000); +fn a_tree_walk_past_the_index_range_rebases_then_matches_a_plain_walk() { + let mut plain = bt_table(); + crate::encoding::fastpath::with_host_tier!(K => plain.bt_update_tree_until::(40, 64)); + + let mut t = bt_table(); + t.index_shift = u32::MAX as usize - 10; + assert!( + !t.can_skip_rebase_check(40), + "fixture precondition: the target must not be representable", + ); + crate::encoding::fastpath::with_host_tier!(K => t.bt_update_tree_until::(40, 64)); + + assert_eq!(t.index_shift, 0, "the positions were re-encoded"); + assert_eq!(t.skip_insert_until_abs, 40); assert_eq!( - t.skip_insert_until_abs, 100, - "HC mode must not adjust skip cursor" + (t.hash_table(), t.chain_table()), + (plain.hash_table(), plain.chain_table()), + "the position-at-a-time walk built a different tree", ); } #[test] fn apply_limited_update_after_long_match_bt_mode_caps_gap_at_384() { - // BT mode with gap > 384 → cap the skip cursor so future - // `bt_update_tree_until` doesn't walk an unbounded prefix. + // A gap > 384 → cap the skip cursor so future `bt_update_tree_until` + // doesn't walk an unbounded prefix. let mut t = new_table(32); - t.uses_bt = true; t.skip_insert_until_abs = 0; // current_abs_start = 1000 → gap = 1000 → cap subtracts // (gap - 384).min(192) = 192, so result is 1000 - 192 = 808. @@ -395,7 +373,6 @@ fn apply_limited_update_after_long_match_bt_mode_caps_gap_at_384() { #[test] fn apply_limited_update_after_long_match_small_gap_is_noop() { let mut t = new_table(32); - t.uses_bt = true; t.skip_insert_until_abs = 800; // gap = 200 < 384 → no change. t.apply_limited_update_after_long_match(1000); diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index e968e0223..e2d40846f 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -70,6 +70,7 @@ pub(crate) mod bt; pub(crate) mod cost_model; pub(crate) mod dfast; pub(crate) mod hc; +#[cfg(test)] pub(crate) mod lazy_parse; // LDM hashes each `min_match_length` window with XXH64 (upstream zstd // `zstd_ldm.c:315`), so the `ldm` feature implies `hash` and the @@ -111,7 +112,7 @@ pub use levels::config::{ pub use match_generator::MatchGeneratorDriver; pub use parameters::{ Bounds, CParameter, CompressionParameters, CompressionParametersBuilder, LevelParameters, - LiteralCompressionMode, ParameterError, Strategy, + LiteralCompressionMode, ParameterError, RowMatchFinderMode, Strategy, }; pub use streaming_encoder::{CompressionContext, StreamingEncoder}; pub use workspace::HistoryBuf; @@ -537,6 +538,15 @@ pub trait Matcher { /// holds anything there. #[doc(hidden)] fn leave_workspace(&mut self) {} + /// The built-in matcher, when this is it. The encoder then runs its + /// backends compiled for the kernel tier it chose for the block, rather + /// than through [`start_matching`](Self::start_matching), which leaves the + /// tier to the matcher. Every other matcher keeps the default. + #[doc(hidden)] + #[inline] + fn builtin_driver(&mut self) -> Option<&mut match_generator::MatchGeneratorDriver> { + None + } /// Provide a hint about the total uncompressed size for the next frame. /// /// Implementations may use this to select smaller hash tables and windows diff --git a/zstd/src/encoding/opt/types.rs b/zstd/src/encoding/opt/types.rs index 243bf38e7..a8361e146 100644 --- a/zstd/src/encoding/opt/types.rs +++ b/zstd/src/encoding/opt/types.rs @@ -18,6 +18,7 @@ use super::super::cost_model::HcOptimalCostProfile; /// One candidate match produced by a match-finder probe. Carries the /// absolute starting position and the upstream zstd-style offset/length pair. #[derive(Copy, Clone, Debug)] +#[cfg_attr(test, derive(PartialEq))] pub(crate) struct MatchCandidate { pub(crate) start: usize, pub(crate) offset: usize, diff --git a/zstd/src/encoding/parameters.rs b/zstd/src/encoding/parameters.rs index 024307fd9..b9b1d9aa8 100644 --- a/zstd/src/encoding/parameters.rs +++ b/zstd/src/encoding/parameters.rs @@ -154,6 +154,35 @@ pub enum LiteralCompressionMode { Disable, } +/// Which finder the greedy, lazy and lazy2 strategies search, the drop-in +/// equivalent of C zstd's `ZSTD_c_useRowMatchFinder` (`ZSTD_ps_auto` / +/// `ZSTD_ps_enable` / `ZSTD_ps_disable`). The other strategies have a finder +/// of their own and ignore it. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub enum RowMatchFinderMode { + /// Rows above a 16 KiB window, a hash chain at or below it, where the rows + /// cost more to set up than they save (upstream + /// `ZSTD_resolveRowMatchFinderMode`, zstd_compress.c:246). + #[default] + Auto, + /// Search rows at every window size. + Enable, + /// Search a hash chain at every window size. + Disable, +} + +impl RowMatchFinderMode { + /// Whether a greedy / lazy / lazy2 frame with a `window`-byte window + /// searches rows. + pub(crate) const fn searches_rows(self, window: usize) -> bool { + match self { + Self::Auto => window > (1 << 14), + Self::Enable => true, + Self::Disable => false, + } + } +} + /// One tunable compression parameter — the analogue of a C zstd /// `ZSTD_cParameter`. Used to query bounds via [`CParameter::bounds`]. #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -435,6 +464,8 @@ pub(crate) struct ParamOverrides { pub(crate) ldm: Option, /// Whether literals are entropy-coded; `Auto` leaves it to the level. pub(crate) literal_compression: LiteralCompressionMode, + /// The greedy / lazy band's finder; `Auto` leaves it to the window. + pub(crate) row_match_finder: RowMatchFinderMode, } impl ParamOverrides { @@ -451,6 +482,7 @@ impl ParamOverrides { && self.strategy.is_none() && self.ldm.is_none() && self.literal_compression == LiteralCompressionMode::Auto + && self.row_match_finder == RowMatchFinderMode::Auto } } @@ -483,6 +515,7 @@ impl CompressionParameters { enable_ldm: false, ldm: LdmOverride::default(), literal_compression: LiteralCompressionMode::Auto, + row_match_finder: RowMatchFinderMode::Auto, } } @@ -501,6 +534,11 @@ impl CompressionParameters { self.overrides.literal_compression } + /// Which finder the greedy / lazy band searches (see [`RowMatchFinderMode`]). + pub fn row_match_finder_mode(&self) -> RowMatchFinderMode { + self.overrides.row_match_finder + } + /// The window, `log2`, the encoder gives a frame of `source_size` bytes /// (`None` = unknown) with `dictionary_size` bytes of dictionary content /// under these parameters: resolved exactly as the encoder resolves it. @@ -580,9 +618,37 @@ pub struct CompressionParametersBuilder { enable_ldm: bool, ldm: LdmOverride, literal_compression: LiteralCompressionMode, + row_match_finder: RowMatchFinderMode, } impl CompressionParametersBuilder { + /// Choose the finder the greedy, lazy and lazy2 strategies search instead + /// of leaving it to the window size. C `ZSTD_c_useRowMatchFinder`. + /// + /// ```rust + /// use structured_zstd::encoding::{ + /// compress_with_parameters, CompressionLevel, CompressionParameters, + /// RowMatchFinderMode, + /// }; + /// use structured_zstd::decoding::FrameDecoder; + /// + /// let data = b"a row or a chain, the frame decodes the same ".repeat(2000); + /// let chain = CompressionParameters::builder(CompressionLevel::Level(7)) + /// .row_match_finder(RowMatchFinderMode::Disable) + /// .build() + /// .unwrap(); + /// let frame = compress_with_parameters(&data[..], &chain); + /// let mut decoded = Vec::with_capacity(data.len()); + /// FrameDecoder::new() + /// .decode_all_to_vec(&frame, &mut decoded) + /// .unwrap(); + /// assert_eq!(decoded, data); + /// ``` + pub fn row_match_finder(mut self, mode: RowMatchFinderMode) -> Self { + self.row_match_finder = mode; + self + } + /// Decide whether literals are entropy-coded, overriding the level's own /// choice. C `ZSTD_c_literalCompressionMode`. /// @@ -741,6 +807,7 @@ impl CompressionParametersBuilder { strategy: self.strategy, ldm, literal_compression: self.literal_compression, + row_match_finder: self.row_match_finder, }, }) } diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index e977388b4..0bd6215c2 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -287,20 +287,21 @@ impl RowDictShape { && self.use_bt == other.use_bt } } +use super::match_table::helpers::INCOMPRESSIBLE_SKIP_STEP; +#[cfg(test)] use super::match_table::helpers::{ - INCOMPRESSIBLE_SKIP_STEP, best_len_offset_candidate, extend_backwards_shared, - repcode_candidate_shared, + best_len_offset_candidate, extend_backwards_shared, repcode_candidate_shared, }; use super::match_table::storage::REBASE_RESET_FLOOR_CEILING; +#[cfg(test)] use super::opt::types::MatchCandidate; -// The row probe reuses the shared `fastpath::FastpathKernel` selection so each -// per-tier `#[target_feature]` probe can inline BOTH the tag-match mask AND the -// matching `fastpath::::common_prefix_len_ptr` (the tiers must share a -// feature set for the cpl to inline — `Sse42` is a superset of the SSE2 mask -// intrinsics, `Avx2Bmi2` of the AVX2 mask). `select_kernel()` does the runtime -// detect once per process via a `OnceLock`. -use super::fastpath::FastpathKernel; +// The row probe runs under the encoder's tier so each per-tier +// `#[target_feature]` probe can inline BOTH the tag-match mask AND the matching +// `fastpath::::common_prefix_len_ptr` (the tiers must share a feature set +// for the cpl to inline — `Sse42` is a superset of the SSE2 mask intrinsics, +// `Avx2Bmi2` of the AVX2 mask). +use super::fastpath::{EncodeTier, FastpathKernel}; /// Compile-time row tag-match kernel. Each ZST monomorphises the per-row /// tag compare so the search hot loop drops the runtime `RowTagKernel` @@ -311,6 +312,10 @@ use super::fastpath::FastpathKernel; /// supports its ISA — the same contract `RowTagKernel::detect` upholds for /// the enum's `unsafe` SIMD calls. pub(crate) trait RowTags: Copy { + /// The encoder tier this tag kernel runs under, whose byte compare the + /// probe's callers use. + type Tier: EncodeTier; + /// Run the row match probe (live row + dict dual-probe) for this kernel. /// Forwards to the matcher's per-tier `#[target_feature]` probe method whose /// body expands the tier's `row_tag_mask_*!` SIMD inline (no function call), @@ -324,6 +329,7 @@ pub(crate) trait RowTags: Copy { /// The caller (via `dispatch_tag_kernel!`) only selects a kernel whose ISA /// `RowTagKernel::detect` confirmed present, upholding the per-tier /// `#[target_feature]` contract. + #[cfg(test)] unsafe fn probe( matcher: &RowMatchGenerator, abs_pos: usize, @@ -346,6 +352,9 @@ pub(crate) trait RowTags: Copy { #[derive(Copy, Clone)] struct ScalarTags; impl RowTags for ScalarTags { + type Tier = crate::encoding::fastpath::ScalarTier; + + #[cfg(test)] #[inline] unsafe fn probe( matcher: &RowMatchGenerator, @@ -369,6 +378,10 @@ struct Sse2Tags; feature = "kernel-sse" ))] impl RowTags for Sse2Tags { + // `Sse42` shares this tag kernel and the SSE2 compare. + type Tier = crate::encoding::fastpath::Sse2Tier; + + #[cfg(test)] #[inline] unsafe fn probe( matcher: &RowMatchGenerator, @@ -394,6 +407,9 @@ struct Avx2Bmi2Tags; feature = "kernel-avx2" ))] impl RowTags for Avx2Bmi2Tags { + type Tier = crate::encoding::fastpath::Avx2Bmi2Tier; + + #[cfg(test)] #[inline] unsafe fn probe( matcher: &RowMatchGenerator, @@ -419,6 +435,9 @@ struct NeonTags; feature = "kernel-neon" ))] impl RowTags for NeonTags { + type Tier = crate::encoding::fastpath::NeonTier; + + #[cfg(test)] #[inline] unsafe fn probe( matcher: &RowMatchGenerator, @@ -448,6 +467,9 @@ struct Simd128Tags; feature = "kernel-simd128" ))] impl RowTags for Simd128Tags { + type Tier = crate::encoding::fastpath::Simd128Tier; + + #[cfg(test)] #[inline] unsafe fn probe( matcher: &RowMatchGenerator, @@ -461,12 +483,13 @@ impl RowTags for Simd128Tags { } } -/// Resolve the runtime `tag_kernel` (`FastpathKernel`) to a `RowTags` ZST once, -/// then call a `*_k::` method that binds the `row_log` const. The kernel -/// branch runs once per block (cold), so the per-position hot loop is fully -/// monomorphised over the selected tier — no runtime kernel enum inside. +/// Map the encoder tier `$tier` to its `RowTags` ZST, then call a `*_k::` +/// method that binds the `row_log` const. The match on the tier's constant +/// folds at monomorphisation, so the per-position hot loop is fully +/// monomorphised over the tier with no kernel choice anywhere. +#[cfg(test)] macro_rules! dispatch_tag_kernel { - ($self:ident . $k_method:ident ( $($arg:expr),* )) => {{ + ($tier:ty, $self:ident . $k_method:ident ( $($arg:expr),* )) => {{ // wasm32 has no runtime CPUID: when the build enables `simd128`, the // tier is resolved at compile time straight to `Simd128Tags`, so the // runtime `FastpathKernel` match (native-only) is cfg'd out entirely. @@ -484,7 +507,7 @@ macro_rules! dispatch_tag_kernel { feature = "kernel-simd128" )))] { - match $self.tag_kernel { + crate::encoding::fastpath::tier_match!($tier { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-avx2" @@ -496,7 +519,7 @@ macro_rules! dispatch_tag_kernel { ))] FastpathKernel::Sse2 | FastpathKernel::Sse42 => { $self.$k_method::($($arg),*) - } + }, #[cfg(all( target_arch = "aarch64", target_endian = "little", @@ -504,7 +527,7 @@ macro_rules! dispatch_tag_kernel { ))] FastpathKernel::Neon => $self.$k_method::($($arg),*), FastpathKernel::Scalar => $self.$k_method::($($arg),*), - } + }) } }}; } @@ -1802,6 +1825,7 @@ macro_rules! dispatch_lazy { /// Bind the runtime `row_log` (clamped 4..=6) to the const `ROW_LOG` of a /// `*_rl::` hot loop. Mirrors the upstream zstd's per-rowLog variant /// table; the branch is cold (once per block / call). +#[cfg(test)] macro_rules! dispatch_row_log { ($self:ident . $rl_method:ident :: <$k:ty> ( $($arg:expr),* )) => { match $self.row_log { @@ -1990,6 +2014,7 @@ macro_rules! row_tag_mask_simd128 { /// spills per position. Early exits use the labeled block (`break 'probe`) /// because a `return` inside a macro body would return from the EXPANSION /// SITE's function. +#[cfg(test)] macro_rules! row_probe_body { ($m:expr, $abs_pos:expr, $lit_len:expr, $hash:expr, $seed:expr, $rl:expr, $use_mask:literal, $maskmac:ident, $cpl:path) => {{ #[allow(unused_labels)] @@ -2289,6 +2314,7 @@ macro_rules! row_probe_body { }}; } +#[cfg(test)] macro_rules! gen_row_probe { ($name:ident, $use_mask:literal, $maskmac:ident, $cpl:path $(, $tf:literal)?) => { $(#[target_feature(enable = $tf)])? @@ -2466,10 +2492,6 @@ pub(crate) struct RowMatchGenerator { /// indexing, `0` for all of it (see [`Self::set_dictionary_index_floor`]). dict_index_floor: usize, pub(crate) lazy_depth: u8, - /// Cached fastpath kernel for the per-candidate `common_prefix_len` - /// compare, resolved once per matcher so the rep probe skips the - /// `select_kernel()` atomic on every input byte. - pub(crate) cpl_kernel: crate::encoding::fastpath::FastpathKernel, /// Single backing allocation for every `u32` index table this matcher /// owns (the cwksp analog the dfast backend already uses for its long / /// short pair). The row finder and the chain / tree finder are mutually @@ -2516,19 +2538,6 @@ pub(crate) struct RowMatchGenerator { // [`Self::rebase_positions`]), keeping positions representable without // capping frame length. The tags and the slot cursors live in the byte // tail of that same buffer; see [`Self::rows_len`]. - /// Cached tag-match SIMD kernel; CPU features are fixed per process, so - /// resolve once instead of querying per `row_candidate` call. On - /// wasm32+simd128 the tier is compile-time (`dispatch_tag_kernel!` selects - /// `Simd128Tags` directly), so the field is unread there. - #[cfg_attr( - all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ), - allow(dead_code) - )] - tag_kernel: FastpathKernel, /// Attached immutable dictionary row index (upstream zstd `dictMatchState`). `Some` /// activates the bounded dict probe in `row_candidate_rl`; built once and /// cached across frames via `DictAttach`, invalidated on eviction / resize. @@ -2603,11 +2612,9 @@ impl RowMatchGenerator { lazy_reps: None, dict_index_floor: 0, lazy_depth: 1, - cpl_kernel: crate::encoding::fastpath::select_kernel(), rows_len: 0, heads_len: 0, tags_len: 0, - tag_kernel: crate::encoding::fastpath::select_kernel(), dict: DictAttach::new(), dict_keep: DictKeep::new(), #[cfg(test)] @@ -2778,16 +2785,17 @@ impl RowMatchGenerator { self.row_hash_mls = self.mls.clamp(4, 6) as u32; // A btlazy2 level (its own or the CDict's strategy) searches the // binary tree. Otherwise upstream `ZSTD_resolveRowMatchFinderMode`: - // rows only above a 2^14 window; the window is source-size-adjusted, - // so inputs of 16 KiB or less search the hash chain. A dictionary - // frame inherits the CDict's decision (`ZSTD_resetCCtx_usingCDict`), - // and a copied dictionary brings the CDict's salt (0) with its tables. + // the mode the caller set, or rows only above a 2^14 window; the + // window is source-size-adjusted, so inputs of 16 KiB or less search + // the hash chain. A dictionary frame inherits the CDict's decision + // (`ZSTD_resetCCtx_usingCDict`), and a copied dictionary brings the + // CDict's salt (0) with its tables. self.finder = if config.bt { LazyFinder::Tree } else { let rows = match self.dict_plan { Some(plan) => plan.use_row, - None => self.max_window_size > (1usize << 14), + None => config.searches_rows(self.max_window_size), }; if rows { LazyFinder::Rows @@ -3374,7 +3382,9 @@ impl RowMatchGenerator { /// deviate here — both the rep probe at `abs_pos + 1` *and* the /// regular `row_candidate(abs_pos, ..)` are evaluated each /// iteration, and the longer match wins (ties go to rep for - /// cheaper encoding via [`best_len_offset_candidate`]). Upstream zstd + /// cheaper encoding via + /// [`best_len_offset_candidate`](super::match_table::helpers::best_len_offset_candidate)). + /// Upstream zstd /// can afford pure commit-on-first-rep because it recovers any /// ratio loss via superblock-level entropy sharing, which we /// don't replicate yet, so the hybrid form avoids a measured @@ -3397,7 +3407,8 @@ impl RowMatchGenerator { /// `offset_2` for back-to-back hits. It is omitted here: the /// main-loop rep probe at `abs_pos + 1` already evaluates all /// three rep slots (rep1, rep2, rep3 + the upstream zstd `ll0` fallback) - /// via [`repcode_candidate_shared`], so the inner-loop slot + /// via [`repcode_candidate_shared`](super::match_table::helpers::repcode_candidate_shared), + /// so the inner-loop slot /// upstream zstd's single-rep design would catch is already covered by /// the next main-loop iteration. Confirmed dead-on-arrival via a /// `panic!` probe across the full 528-test suite + benchmark @@ -3726,6 +3737,7 @@ impl RowMatchGenerator { &self.history[self.history_start..self.history.len() - self.uncommitted_len] } + #[cfg(test)] fn history_abs_end(&self) -> usize { self.history_abs_start + self.live_history().len() } @@ -3925,22 +3937,23 @@ impl RowMatchGenerator { .max(self.history_abs_start) } - /// Used only by the dead-code [`Self::start_matching`] (lazy-style - /// row parse). Kept paired with that method so reviving the lazy - /// path doesn't have to re-derive the rep+row best-of-two pick. + /// The rep+row best-of-two pick at one position, for unit tests that probe + /// the row search outside the production monolith. + #[cfg(test)] #[inline(always)] pub(crate) fn best_match_rl( &self, abs_pos: usize, lit_len: usize, ) -> Option { - let rep = self.repcode_candidate(abs_pos, lit_len); + let rep = self.repcode_candidate::(abs_pos, lit_len); // SAFETY: `K` selected by `dispatch_tag_kernel!` after `detect` confirmed // its ISA; `K::probe` upholds the per-tier feature contract. let row = unsafe { K::probe::(self, abs_pos, lit_len, None) }; best_len_offset_candidate(rep, row) } + #[cfg(test)] #[inline(always)] pub(crate) fn pick_lazy_match_rl( &self, @@ -3970,15 +3983,14 @@ impl RowMatchGenerator { } } - #[allow(dead_code)] + #[cfg(test)] #[inline(always)] - pub(crate) fn repcode_candidate( + pub(crate) fn repcode_candidate( &self, abs_pos: usize, lit_len: usize, ) -> Option { - repcode_candidate_shared( - self.cpl_kernel, + repcode_candidate_shared::( self.live_history(), self.history_abs_start, self.offset_hist, @@ -3995,53 +4007,47 @@ impl RowMatchGenerator { // (driver + tests) use these bare names; the hot loops call the `_rl` // siblings directly with the type and const already bound. `skip` does // no tag compare, so it dispatches on `row_log` only. - pub(crate) fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { - // SAFETY: same per-tier umbrella contract as `start_matching_greedy`. - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - { - // SAFETY: simd128 is a compile-time feature here; no runtime gate. - unsafe { dispatch_lazy!(self.lazy_simd128::(handle_sequence)) } - } - #[cfg(not(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - )))] - { - match self.tag_kernel { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" - ))] - FastpathKernel::Avx2Bmi2 => unsafe { - dispatch_lazy!(self.lazy_avx2bmi2::(handle_sequence)) - }, - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" - ))] - FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { - dispatch_lazy!(self.lazy_sse2::(handle_sequence)) - }, - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - FastpathKernel::Neon => unsafe { - dispatch_lazy!(self.lazy_neon::(handle_sequence)) - }, - // SAFETY: the scalar kernel has no `#[target_feature]`; the - // fn is `unsafe` only for macro uniformity. - FastpathKernel::Scalar => unsafe { - dispatch_lazy!(self.lazy_scalar::(handle_sequence)) - }, - } - } + pub(crate) fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { + // SAFETY: same per-tier umbrella contract as `start_matching_greedy`: + // `K` names a tier the encoder confirmed this CPU runs. + crate::encoding::fastpath::tier_match!(K { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] + FastpathKernel::Avx2Bmi2 => unsafe { + dispatch_lazy!(self.lazy_avx2bmi2::(handle_sequence)) + }, + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" + ))] + FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { + dispatch_lazy!(self.lazy_sse2::(handle_sequence)) + }, + #[cfg(all( + target_arch = "aarch64", + target_endian = "little", + feature = "kernel-neon" + ))] + FastpathKernel::Neon => unsafe { + dispatch_lazy!(self.lazy_neon::(handle_sequence)) + }, + // simd128 is a compile-time feature on wasm; no runtime gate. + #[cfg(all( + target_arch = "wasm32", + target_feature = "simd128", + feature = "kernel-simd128" + ))] + FastpathKernel::Simd128 => unsafe { + dispatch_lazy!(self.lazy_simd128::(handle_sequence)) + }, + // SAFETY: the scalar kernel has no `#[target_feature]`; the + // fn is `unsafe` only for macro uniformity. + FastpathKernel::Scalar => unsafe { + dispatch_lazy!(self.lazy_scalar::(handle_sequence)) + }, + }) } gen_lazy_monolith!( @@ -4161,7 +4167,7 @@ impl RowMatchGenerator { /// into the owned mirror). `history_abs_start` was forced /// to 0 in `set_borrowed_window`, so positions stay absolute input /// offsets and the window-low candidate cap bounds offsets to the window. - pub(crate) fn start_matching_borrowed( + pub(crate) fn start_matching_borrowed( &mut self, block_start: usize, block_end: usize, @@ -4171,7 +4177,7 @@ impl RowMatchGenerator { self.stage_borrowed_block(block_start, block_end); // Greedy runs the same lazy monolith at depth 0. let _ = greedy; - self.start_matching(handle_sequence); + self.start_matching::(handle_sequence); } /// Borrowed equivalent of [`Self::skip_matching_with_hint`]: stage the @@ -4187,23 +4193,39 @@ impl RowMatchGenerator { self.skip_matching_with_hint(incompressible_hint); } - #[allow(dead_code)] + /// The best match at `abs_pos`, which every tier the CPU runs must find + /// alike. + #[cfg(test)] pub(crate) fn best_match(&self, abs_pos: usize, lit_len: usize) -> Option { - dispatch_tag_kernel!(self.best_match_k(abs_pos, lit_len)) + crate::encoding::fastpath::same_under_every_tier(|tier| { + crate::encoding::fastpath::with_encode_tier!( + tier, + T => dispatch_tag_kernel!(T, self.best_match_k(abs_pos, lit_len)) + ) + }) } + #[cfg(test)] fn best_match_k(&self, abs_pos: usize, lit_len: usize) -> Option { dispatch_row_log!(self.best_match_rl::(abs_pos, lit_len)) } - #[allow(dead_code)] + /// The lazy pick at `abs_pos`, which every tier the CPU runs must make + /// alike. + #[cfg(test)] pub(crate) fn pick_lazy_match( &self, abs_pos: usize, lit_len: usize, best: Option, ) -> Option { - dispatch_tag_kernel!(self.pick_lazy_match_k(abs_pos, lit_len, best)) + crate::encoding::fastpath::same_under_every_tier(|tier| { + crate::encoding::fastpath::with_encode_tier!( + tier, + T => dispatch_tag_kernel!(T, self.pick_lazy_match_k(abs_pos, lit_len, best)) + ) + }) } + #[cfg(test)] fn pick_lazy_match_k( &self, abs_pos: usize, @@ -4217,10 +4239,16 @@ impl RowMatchGenerator { // `dispatch_tag_kernel!`; the selected tier's `row_probe_*` method is the // monomorphised per-position hot loop with the SIMD tag-match inlined under // its `#[target_feature]` umbrella (no dispatcher branch inside the loop). - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn row_candidate(&self, abs_pos: usize, lit_len: usize) -> Option { - dispatch_tag_kernel!(self.row_candidate_k(abs_pos, lit_len)) + crate::encoding::fastpath::same_under_every_tier(|tier| { + crate::encoding::fastpath::with_encode_tier!( + tier, + T => dispatch_tag_kernel!(T, self.row_candidate_k(abs_pos, lit_len)) + ) + }) } + #[cfg(test)] fn row_candidate_k( &self, abs_pos: usize, @@ -4242,6 +4270,8 @@ impl RowMatchGenerator { // probe's: the SSE2 umbrella covers the 128-bit mask intrinsics, // AVX2+BMI2 ⊇ the AVX2 mask). // Scalar uses the on-the-fly per-slot byte compare (`use_mask = false`). + // The probes serve the unit tests' row search; production runs the monolith. + #[cfg(test)] gen_row_probe!( row_probe_scalar, false, @@ -4249,6 +4279,7 @@ impl RowMatchGenerator { crate::encoding::fastpath::scalar::common_prefix_len_ptr ); #[cfg(all( + test, any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-sse" ))] @@ -4260,6 +4291,7 @@ impl RowMatchGenerator { "sse2" ); #[cfg(all( + test, any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-avx2" ))] @@ -4271,6 +4303,7 @@ impl RowMatchGenerator { "avx2,bmi2" ); #[cfg(all( + test, target_arch = "aarch64", target_endian = "little", feature = "kernel-neon" @@ -4287,6 +4320,7 @@ impl RowMatchGenerator { // wasm tier's behavior of vectorising only the tag scan, with the portable // prefix-length kernel. #[cfg(all( + test, target_arch = "wasm32", target_feature = "simd128", feature = "kernel-simd128" @@ -4298,6 +4332,7 @@ impl RowMatchGenerator { crate::encoding::fastpath::scalar::common_prefix_len_ptr ); + #[cfg(test)] fn extend_backwards( &self, candidate_pos: usize, diff --git a/zstd/src/encoding/simple/fast_kernel/kernel.rs b/zstd/src/encoding/simple/fast_kernel/kernel.rs index fcf1e72bf..573c51060 100644 --- a/zstd/src/encoding/simple/fast_kernel/kernel.rs +++ b/zstd/src/encoding/simple/fast_kernel/kernel.rs @@ -2050,12 +2050,15 @@ fast_dict_borrowed_wrapper!( crate::encoding::fastpath::simd128::common_prefix_len_ptr ); -/// Dispatch the borrowed dual-base dict kernel to the resolved per-tier SIMD -/// wrapper. `kernel` is the matcher's once-resolved -/// [`FastpathKernel`](crate::encoding::fastpath::FastpathKernel); the -/// scalar arm carries no `#[target_feature]` (generic word-at-a-time count). +/// The borrowed dual-base dict kernel compiled for the encoder tier `K`: its +/// per-tier SIMD wrapper, or for the scalar tier the plain body (generic +/// word-at-a-time count, no `#[target_feature]`). #[allow(clippy::too_many_arguments)] -pub(crate) fn compress_block_fast_dict_borrowed( +pub(crate) fn compress_block_fast_dict_borrowed< + K: crate::encoding::fastpath::EncodeTier, + const MLS: u32, + const USE_CMOV: bool, +>( inp: &[u8], dict: &[u8], block_start: usize, @@ -2066,7 +2069,6 @@ pub(crate) fn compress_block_fast_dict_borrowed FastBlockResult { // Used by the per-tier match arms below; on a target with no SIMD tier // (e.g. wasm32 without simd128) only the scalar fallback compiles and the @@ -2099,7 +2101,7 @@ pub(crate) fn compress_block_fast_dict_borrowed unsafe { cbfd_borrowed_avx2_bmi2::( @@ -2133,8 +2135,8 @@ pub(crate) fn compress_block_fast_dict_borrowed { let mut hs = handle_sequence; scalar(inp, dict, main_table, dict_table, &mut hs) - } - } + }, + }) } #[cfg(all( target_arch = "aarch64", @@ -2142,7 +2144,7 @@ pub(crate) fn compress_block_fast_dict_borrowed unsafe { cbfd_borrowed_neon::( inp, @@ -2160,8 +2162,8 @@ pub(crate) fn compress_block_fast_dict_borrowed { let mut hs = handle_sequence; scalar(inp, dict, main_table, dict_table, &mut hs) - } - } + }, + }) } #[cfg(all( target_arch = "wasm32", @@ -2169,7 +2171,7 @@ pub(crate) fn compress_block_fast_dict_borrowed unsafe { cbfd_borrowed_simd128::( inp, @@ -2187,8 +2189,8 @@ pub(crate) fn compress_block_fast_dict_borrowed { let mut hs = handle_sequence; scalar(inp, dict, main_table, dict_table, &mut hs) - } - } + }, + }) } #[cfg(not(any( target_arch = "x86", @@ -2205,7 +2207,6 @@ pub(crate) fn compress_block_fast_dict_borrowed( + let result = compress_block_fast_dict_borrowed::( &inp, &dict, 0, @@ -656,7 +656,6 @@ fn borrowed_dict_kernel_reconstructs_via_dual_base() { [0, 0], 2, &mut handle, - FastpathKernel::Scalar, ); // Reconstruct against the logical [dict][input] window: start from the @@ -706,7 +705,7 @@ fn borrowed_dict_kernel_reconstructs_via_dual_base() { /// four-byte gate has to hold there too: the match is real and must be found. #[test] fn borrowed_dict_kernel_takes_a_repcode_whose_candidate_is_in_the_dictionary() { - use crate::encoding::fastpath::FastpathKernel; + use crate::encoding::fastpath::ScalarTier; let hash_log = 12u32; const MLS: u32 = 4; @@ -731,7 +730,7 @@ fn borrowed_dict_kernel_takes_a_repcode_whose_candidate_is_in_the_dictionary() { let mut replay = BlockReplay::new(&inp); let mut handle = |seq: Sequence| tuples.push(capture(&mut replay, seq)); - let result = compress_block_fast_dict_borrowed::( + let result = compress_block_fast_dict_borrowed::( &inp, &dict, 0, @@ -745,7 +744,6 @@ fn borrowed_dict_kernel_takes_a_repcode_whose_candidate_is_in_the_dictionary() { [offset_1, 0], 2, &mut handle, - FastpathKernel::Scalar, ); let mut window = dict.clone(); @@ -783,7 +781,7 @@ fn borrowed_dict_kernel_takes_a_repcode_whose_candidate_is_in_the_dictionary() { /// failed four-byte gate would have rejected it. #[test] fn borrowed_dict_kernel_rejects_a_short_repcode_in_the_dictionary_tail() { - use crate::encoding::fastpath::FastpathKernel; + use crate::encoding::fastpath::ScalarTier; let hash_log = 12u32; const MLS: u32 = 4; @@ -801,7 +799,7 @@ fn borrowed_dict_kernel_rejects_a_short_repcode_in_the_dictionary_tail() { let mut replay = BlockReplay::new(&inp); let mut handle = |seq: Sequence| tuples.push(capture(&mut replay, seq)); - let result = compress_block_fast_dict_borrowed::( + let result = compress_block_fast_dict_borrowed::( &inp, &dict, 0, @@ -815,7 +813,6 @@ fn borrowed_dict_kernel_rejects_a_short_repcode_in_the_dictionary_tail() { [3, 0], 2, &mut handle, - FastpathKernel::Scalar, ); assert!( @@ -836,7 +833,7 @@ fn borrowed_dict_kernel_rejects_a_short_repcode_in_the_dictionary_tail() { /// must still be found, which is what the gate must not cost. #[test] fn borrowed_dict_kernel_finds_a_match_starting_in_the_dictionary_tail() { - use crate::encoding::fastpath::FastpathKernel; + use crate::encoding::fastpath::ScalarTier; let hash_log = 12u32; const MLS: u32 = 4; @@ -868,7 +865,7 @@ fn borrowed_dict_kernel_finds_a_match_starting_in_the_dictionary_tail() { let mut replay = BlockReplay::new(&inp); let mut handle = |seq: Sequence| tuples.push(capture(&mut replay, seq)); - let result = compress_block_fast_dict_borrowed::( + let result = compress_block_fast_dict_borrowed::( &inp, &dict, 0, @@ -882,7 +879,6 @@ fn borrowed_dict_kernel_finds_a_match_starting_in_the_dictionary_tail() { [0, 0], 2, &mut handle, - FastpathKernel::Scalar, ); let mut window = dict.clone(); diff --git a/zstd/src/encoding/simple/fast_matcher.rs b/zstd/src/encoding/simple/fast_matcher.rs index 84b194a39..612f8f134 100644 --- a/zstd/src/encoding/simple/fast_matcher.rs +++ b/zstd/src/encoding/simple/fast_matcher.rs @@ -248,14 +248,6 @@ pub(crate) struct FastKernelMatcher { /// `adjust_params_for_source_size` clamps `window_log` below /// the upstream zstd default of 19, flipping `use_cmov` on). use_cmov: bool, - /// Cached per-tier SIMD kernel selection (resolved once via - /// [`crate::encoding::fastpath::select_kernel`] at construction / reset), - /// mirroring the Dfast/Row backends. Drives the `#[target_feature]` - /// umbrella dispatch in the borrowed dual-base dict scan so the - /// match-length `common_prefix_len_ptr` is the tier's 32-byte AVX2 / - /// 16-byte SSE4.2 / NEON / wasm-simd128 compare instead of the generic - /// word-at-a-time `count_forward`. - kernel: crate::encoding::fastpath::FastpathKernel, /// Initial step the kernel uses for the 4-cursor body's skip /// schedule. Upstream zstd `stepSize = targetLength + !(targetLength) + /// 1` (min 2). Negative-level frames set this to 2..8 to @@ -358,7 +350,6 @@ impl Clone for FastKernelMatcher { max_window_size: self.max_window_size, window_log: self.window_log, use_cmov: self.use_cmov, - kernel: self.kernel, step_size: self.step_size, staged_block: self.staged_block, last_block_start: self.last_block_start, @@ -388,7 +379,6 @@ impl Clone for FastKernelMatcher { self.max_window_size = source.max_window_size; self.window_log = source.window_log; self.use_cmov = source.use_cmov; - self.kernel = source.kernel; self.step_size = source.step_size; self.staged_block = source.staged_block; self.last_block_start = source.last_block_start; @@ -626,7 +616,6 @@ impl FastKernelMatcher { max_window_size: 1usize << window_log, window_log, use_cmov: window_log < 19, - kernel: crate::encoding::fastpath::select_kernel(), step_size, staged_block: None, borrowed: None, @@ -1450,7 +1439,7 @@ impl FastKernelMatcher { /// lookahead, dense fills, backward extension, immediate repcode-2 loop) — /// the prior scalar greedy scan had none of these and was +68% slower. /// Validated by roundtrip + cross-validation + the FFI ratio gate. - pub(crate) fn start_matching_borrowed_dict( + pub(crate) fn start_matching_borrowed_dict( &mut self, block_start: usize, block_end: usize, @@ -1484,7 +1473,6 @@ impl FastKernelMatcher { let use_cmov = self.use_cmov; let step_size = self.step_size; let rep_in = self.rep; - let kernel = self.kernel; // Window bounds in VIRTUAL `[dict][input]` coords, so the kernel's gates // match the owned flat dict kernel: `window_low` is the absolute floor; @@ -1539,7 +1527,7 @@ impl FastKernelMatcher { macro_rules! run { ($mls:literal, $cmov:literal) => { - compress_block_fast_dict_borrowed::<$mls, $cmov>( + compress_block_fast_dict_borrowed::( inp, dict, block_start, @@ -1550,7 +1538,6 @@ impl FastKernelMatcher { rep_in, step_size, &mut handle_sequence, - kernel, ) }; } diff --git a/zstd/src/encoding/strategy.rs b/zstd/src/encoding/strategy.rs index 32ef81d3d..097cc5350 100644 --- a/zstd/src/encoding/strategy.rs +++ b/zstd/src/encoding/strategy.rs @@ -12,9 +12,7 @@ //! Matcher::start_matching // 7-arm match on StrategyTag (per block) //! └─ compress_block:: // S::BACKEND const match //! ├─ Simple/Dfast/Row // backends without parse_mode -//! └─ HcMatchGenerator::start_matching_strategy:: -//! ├─ S::USE_BT == false → start_matching_lazy -//! └─ S::USE_BT == true → start_matching_optimal:: +//! └─ HcMatchGenerator::start_matching_optimal:: //! ├─ HcOptimalCostProfile::const_for_strategy::() //! ├─ should_run_btultra2_seed_pass:: // const false unless S = BtUltra2 //! └─ select_kernel() once, then one pass over the block: @@ -48,7 +46,7 @@ pub(crate) enum BackendTag { Dfast, /// `RowMatchGenerator` — level 4. Row, - /// `HcMatchGenerator` — levels 5-22. + /// `HcMatchGenerator` — the optimal parsers (btopt, btultra, btultra2). HashChain, } @@ -93,19 +91,17 @@ impl ParseMode { } /// Search method — how match candidates are produced for a position. -/// Orthogonal to [`ParseMode`] (upstream zstd `cParams.strategy` search half plus -/// the `useRowMatchFinder` cParam that swaps `HashChain` for `RowHash` in -/// the greedy/lazy band). +/// Orthogonal to [`ParseMode`] (upstream zstd `cParams.strategy` search half). #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub(crate) enum SearchMethod { /// Single-table fast finder (upstream zstd `ZSTD_fast`). Fast, /// Two parallel hash tables (upstream zstd `ZSTD_dfast`). DoubleFast, - /// Row-hash SIMD-tag finder (upstream zstd row matchfinder). + /// The greedy / lazy / lazy2 finders: rows or the hash chain + /// (`ZSTD_RowFindBestMatch` / `ZSTD_HcFindBestMatch`, chosen by + /// `useRowMatchFinder`), searched by one lazy parse. RowHash, - /// Hash-chain finder (upstream zstd `ZSTD_HcFindBestMatch`). - HashChain, /// Binary-tree finder for the optimal parser (upstream zstd `ZSTD_BtGetAllMatches`). BinaryTree, /// Lazily-sorted binary-tree finder inside the lazy parse (upstream zstd @@ -115,15 +111,14 @@ pub(crate) enum SearchMethod { impl SearchMethod { /// Storage backend (matcher variant) this search method runs in. - /// `BinaryTree` shares the `HashChain` storage (the BT scratch lives - /// inside the hash-chain matcher, gated by `uses_bt`); `BinaryTreeLazy` - /// is the third finder of the lazy (Row) backend. + /// `BinaryTree` runs in the `HashChain` storage, whose tables hold the + /// tree; `BinaryTreeLazy` is the third finder of the lazy (Row) backend. pub(crate) const fn backend(self) -> BackendTag { match self { Self::Fast => BackendTag::Simple, Self::DoubleFast => BackendTag::Dfast, Self::RowHash | Self::BinaryTreeLazy => BackendTag::Row, - Self::HashChain | Self::BinaryTree => BackendTag::HashChain, + Self::BinaryTree => BackendTag::HashChain, } } } diff --git a/zstd/src/encoding/streaming_encoder.rs b/zstd/src/encoding/streaming_encoder.rs index 069fcdb94..ba0fe025f 100644 --- a/zstd/src/encoding/streaming_encoder.rs +++ b/zstd/src/encoding/streaming_encoder.rs @@ -10,6 +10,7 @@ use core::hash::Hasher; #[cfg(feature = "hash")] use twox_hash::XxHash64; +use crate::encoding::fastpath::EncodeTier; use crate::encoding::levels::compress_block_encoded; use crate::encoding::{ CompressionLevel, EncoderDictionary, MatchGeneratorDriver, Matcher, block_header::BlockHeader, @@ -725,6 +726,20 @@ impl CompressionContext { /// set with [`set_pledged_content_size`](Self::set_pledged_content_size) /// allows less. pub fn write(&mut self, drain: &mut D, buf: &[u8]) -> Result { + // The call's one choice of kernel tier: the frame start and every block + // it emits run compiled for it. + crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => self.write_with::(drain, buf) + ) + } + + /// [`Self::write`] compiled for the kernel tier `K`. + fn write_with( + &mut self, + drain: &mut D, + buf: &[u8], + ) -> Result { self.ensure_open()?; if buf.is_empty() { return Ok(0); @@ -741,7 +756,7 @@ impl CompressionContext { )); } - self.ensure_frame_started(drain)?; + self.ensure_frame_started::(drain)?; // Enforce pledged upper bound: truncate the accepted slice to the // remaining allowance so that partial-write semantics are honored @@ -773,7 +788,10 @@ impl CompressionContext { remaining = &remaining[to_take..]; consumed += to_take; - if self.emit_full_pending_block(drain, block_capacity).is_err() { + if self + .emit_full_pending_block::(drain, block_capacity) + .is_err() + { // The bytes this call took are reported; the failure, sticky // on the context now, is what the next call returns. break; @@ -789,8 +807,13 @@ impl CompressionContext { if self.pending_len() == 0 { return drain.flush().map_err(|err| self.fail(err)); } - self.ensure_frame_started(drain)?; - self.emit_pending_block(drain, false)?; + crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => { + self.ensure_frame_started::(drain)?; + self.emit_pending_block::(drain, false) + } + )?; drain.flush().map_err(|err| self.fail(err)) } @@ -812,10 +835,15 @@ impl CompressionContext { )); } - self.ensure_frame_started(drain)?; // An empty frame, or one whose input ended on a block boundary, closes // with an empty last block. - self.emit_pending_block(drain, true)?; + crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => { + self.ensure_frame_started::(drain)?; + self.emit_pending_block::(drain, true) + } + )?; #[cfg(feature = "hash")] if self.content_checksum { @@ -881,7 +909,10 @@ impl CompressionContext { /// ``` pub fn start_frame(&mut self, drain: &mut D) -> Result<(), Error> { self.ensure_open()?; - self.ensure_frame_started(drain) + crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => self.ensure_frame_started::(drain) + ) } /// The largest block the open frame cuts: the block size limits held to @@ -935,7 +966,11 @@ impl CompressionContext { // so the post-split, which would emit several physical blocks, stays // off (upstream `ZSTD_compressBlock` never splits either). let post_split = core::mem::replace(&mut self.state.post_split_allowed, false); - let encoded = self.encode_block(payload, block.len(), false); + // The context's one choice of kernel tier for this block. + let encoded = crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => self.encode_block::<_, K>(payload, block.len(), false) + ); self.state.post_split_allowed = post_split; encoded.map_err(|err| self.fail(err))?; self.bytes_consumed += block.len() as u64; @@ -1021,7 +1056,10 @@ impl CompressionContext { } } - fn ensure_frame_started(&mut self, drain: &mut D) -> Result<(), Error> { + fn ensure_frame_started( + &mut self, + drain: &mut D, + ) -> Result<(), Error> { if self.frame_started { return Ok(()); } @@ -1119,7 +1157,7 @@ impl CompressionContext { self.state.offset_hist = [1, 4, 8]; if use_dictionary_state && let Some(dict) = self.dictionary.as_ref() { self.state.offset_hist = dict.inner.offset_hist; - crate::encoding::frame_compressor::load_frame_dictionary( + crate::encoding::frame_compressor::load_frame_dictionary::( &mut self.state, self.compression_level, dict, @@ -1264,11 +1302,9 @@ impl CompressionContext { /// Where the full pending block is cut (upstream `ZSTD_compress_frameChunk` /// sizing every block with `ZSTD_optimalBlockSize`): the pre-splitter's - /// boundary once the frame has saved enough, else the whole block. - /// `remaining` is the input still to come as far as the splitter knows: - /// a full block again while writes continue, the buffered bytes at the - /// end of the frame. - fn pre_split_len(&mut self, block_capacity: usize, remaining: usize) -> usize { + /// boundary once the frame has saved enough, else the whole block. Writes + /// may continue, so the splitter is told a full block is still to come. + fn pre_split_len(&mut self, block_capacity: usize) -> usize { if matches!(self.compression_level, CompressionLevel::Uncompressed) { return self.pending_len(); } @@ -1277,7 +1313,7 @@ impl CompressionContext { let cut = crate::encoding::frame_compressor::optimal_block_size_with( self.state.pre_split.map(usize::from), pending, - remaining, + block_capacity, block_capacity, self.savings, ) @@ -1285,7 +1321,10 @@ impl CompressionContext { // The splitter sees byte statistics only; a cut it proposes inside a // match the matcher already knows of is moved out. if cut < window { - self.state.matcher.settle_block_cut(window, cut) + match self.state.matcher.builtin_driver() { + Some(driver) => driver.settle_block_cut_with::(window, cut), + None => self.state.matcher.settle_block_cut(window, cut), + } } else { cut } @@ -1294,7 +1333,7 @@ impl CompressionContext { /// Emit the pending block once it is full, cut where the pre-splitter /// says; the suffix stays pending and heads the next block. A failure /// leaves the context failed. - fn emit_full_pending_block( + fn emit_full_pending_block( &mut self, drain: &mut D, block_capacity: usize, @@ -1302,32 +1341,22 @@ impl CompressionContext { if self.pending_len() != block_capacity { return Ok(()); } - let block_len = self.pre_split_len(block_capacity, block_capacity); - self.encode_block(drain, block_len, false) + let block_len = self.pre_split_len::(block_capacity); + self.encode_block::(drain, block_len, false) .map_err(|err| self.fail(err)) } - fn emit_pending_block( + fn emit_pending_block( &mut self, drain: &mut D, last_block: bool, ) -> Result<(), Error> { - let block_capacity = self.block_capacity(); - if last_block { - // A full final buffer is cut like any other block (the reader - // path splits it with `remaining = len`); each cut prefix goes - // out as a non-last block and the suffix is re-examined. - while self.pending_len() == block_capacity { - let block_len = self.pre_split_len(block_capacity, block_capacity); - if block_len == block_capacity { - break; - } - self.encode_block(drain, block_len, false) - .map_err(|err| self.fail(err))?; - } - } + // `write` emits a block the moment it fills, so what is left here is + // always short of one and goes out whole; a write that failed on the + // full block left the context failed, and `ensure_open` stops here. let block_len = self.pending_len(); - self.encode_block(drain, block_len, last_block) + debug_assert!(block_len < self.block_capacity()); + self.encode_block::(drain, block_len, last_block) .map_err(|err| self.fail(err)) } @@ -1347,7 +1376,7 @@ impl CompressionContext { /// Writes the first `block_len` pending bytes to `drain` as one block, /// `last_block` if it closes the frame. - fn encode_block( + fn encode_block( &mut self, drain: &mut D, block_len: usize, @@ -1384,7 +1413,7 @@ impl CompressionContext { } let dict_active = self.dictionary.is_some() && self.state.matcher.supports_dictionary_priming(); - compress_block_encoded( + compress_block_encoded::( &mut self.state, self.compression_level, last_block, diff --git a/zstd/src/encoding/streaming_encoder/tests.rs b/zstd/src/encoding/streaming_encoder/tests.rs index 4ff37d1bb..48b035ae3 100644 --- a/zstd/src/encoding/streaming_encoder/tests.rs +++ b/zstd/src/encoding/streaming_encoder/tests.rs @@ -1960,13 +1960,15 @@ fn a_streamed_raw_frame_carries_a_valid_checksum() { assert_eq!(decoded, payload); } -/// A full final buffer is cut like any other block: where the content changes -/// inside it, the pre-split sends the prefix out as a block of its own before -/// the last one, and the frame still decodes to its input. +/// The buffer the last write fills is cut where its content changes, once the +/// frame has saved enough for the pre-splitter to run: the prefix goes out +/// when the buffer fills and the suffix waits to close the frame. Uncut, the +/// whole buffer would go out with the write and the frame would close on an +/// empty last block. #[test] -fn a_full_last_buffer_is_pre_split_where_its_content_changes() { - let mut payload = b"log line with a steady shape, ".repeat(64 * 1024 / 30); - payload.truncate(64 * 1024); +fn a_buffer_filled_by_the_last_write_is_pre_split_where_its_content_changes() { + let mut payload = b"log line with a steady shape, ".repeat(320 * 1024 / 30 + 1); + payload.truncate(320 * 1024); let mut state = 0x2545_F491_u32; payload.extend((0..64 * 1024).map(|_| { state ^= state << 13; @@ -1977,8 +1979,147 @@ fn a_full_last_buffer_is_pre_split_where_its_content_changes() { let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(19)); encoder.write_all(&payload).unwrap(); let compressed = encoder.finish().unwrap(); + // RFC 8878 3.1.1.2: a 3-byte header (Last_Block, Block_Type, Block_Size) + // then the content: Block_Size bytes, or one byte for an RLE block. + let (_, header_len) = crate::decoding::frame::read_frame_header(&compressed[..]).unwrap(); + let mut at = usize::from(header_len); + let last = loop { + let header = + u32::from_le_bytes([compressed[at], compressed[at + 1], compressed[at + 2], 0]); + let (block_type, size) = ((header >> 1) & 3, (header >> 3) as usize); + at += 3 + if block_type == 1 { 1 } else { size }; + if header & 1 == 1 { + break (block_type, size); + } + }; + assert_ne!( + last, + (0, 0), + "the full buffer went out uncut: the frame closed on an empty block" + ); let mut decoder = StreamingDecoder::new(compressed.as_slice()); let mut decoded = Vec::new(); decoder.read_to_end(&mut decoded).unwrap(); assert_eq!(decoded, payload); } + +/// A drain refusing the block a flush or a finish emits fails that call and +/// leaves the context failed, so a later write reports the failure rather than +/// continuing a frame with a block missing. +#[test] +fn a_drain_refusing_the_pending_block_fails_flush_and_finish() { + for finish in [false, true] { + let mut context = CompressionContext::new(CompressionLevel::Fastest); + // The header goes out with the first write; the block is the second. + let mut drain = FailingWriteOnce::new(2); + context.write(&mut drain, b"ABCD").unwrap(); + let emitted = if finish { + context.finish_frame(&mut drain) + } else { + context.flush(&mut drain) + }; + assert!( + emitted.is_err(), + "finish {finish}: the refused block must fail the call" + ); + assert!( + context.write(&mut drain, b"EFGH").is_err(), + "finish {finish}: the context must stay failed", + ); + } +} + +/// Write `input` in pieces that end off block boundaries, flushing once +/// partway, and finish the frame. +fn stream_in_pieces( + mut encoder: StreamingEncoder, M>, + input: &[u8], +) -> Vec { + for (i, piece) in input.chunks(100_000).enumerate() { + encoder.write_all(piece).unwrap(); + if i == 0 { + encoder.flush().unwrap(); + } + } + encoder.finish().unwrap() +} + +/// What streaming `input` under `tier` writes: frames from the built-in +/// matcher and from one reached only by the trait, and a frame start with one +/// block compressed on its own. +fn streams_under_tier( + tier: crate::encoding::fastpath::FastpathKernel, + level: CompressionLevel, + input: &[u8], + dict: bool, +) -> Vec> { + use crate::encoding::test_support::{TraitDriver, tier_dictionary}; + let dictionary = || EncoderDictionary::from_dictionary(tier_dictionary()); + let mut streams = Vec::new(); + + let mut encoder = StreamingEncoder::new(Vec::new(), level); + encoder.context.state.copy_kernel = tier; + if dict { + encoder.set_encoder_dictionary(dictionary()).unwrap(); + } + streams.push(stream_in_pieces(encoder, input)); + + let mut encoder = StreamingEncoder::new_with_matcher(TraitDriver::new(tier), Vec::new(), level); + encoder.context.state.copy_kernel = tier; + if dict { + encoder.set_encoder_dictionary(dictionary()).unwrap(); + } + streams.push(stream_in_pieces(encoder, input)); + + let mut context = CompressionContext::new(level); + context.state.copy_kernel = tier; + if dict { + context.set_encoder_dictionary(dictionary()).unwrap(); + } + let mut block = Vec::new(); + context.start_frame(&mut block).unwrap(); + assert!( + context + .compress_block(&input[..64 * 1024], &mut block) + .unwrap() + ); + streams.push(block); + streams +} + +/// Every tier the CPU can run streams the same frames, with and without a +/// dictionary, at a level of each backend, and every finished frame decodes +/// to its input: the streaming counterpart of the frame compressor's check, +/// over the write, flush, finish and single-block entry points. +#[test] +fn every_tier_streams_the_same_frames() { + use crate::encoding::test_support::{TIER_LEVELS, tier_dictionary, tier_fixture}; + let input = tier_fixture(); + let tiers = crate::encoding::fastpath::runnable_tiers(); + for level in TIER_LEVELS { + let level = CompressionLevel::Level(level); + for dict in [false, true] { + let reference = streams_under_tier(tiers[0], level, &input, dict); + // The last entry is one block's content, not a frame. + for frame in &reference[..reference.len() - 1] { + let mut decoder = crate::decoding::FrameDecoder::new(); + if dict { + decoder.add_dict(tier_dictionary()).unwrap(); + } + let mut decoded = Vec::with_capacity(input.len()); + decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); + assert!( + decoded == input, + "{level:?} dict {dict}: a frame does not decode to its input" + ); + } + for &tier in &tiers[1..] { + assert!( + streams_under_tier(tier, level, &input, dict) == reference, + "{level:?} dict {dict}: tier {tier:?} streamed other bytes than {:?}", + tiers[0], + ); + } + } + } +} diff --git a/zstd/src/encoding/test_support.rs b/zstd/src/encoding/test_support.rs index 9f800faae..5a85957df 100644 --- a/zstd/src/encoding/test_support.rs +++ b/zstd/src/encoding/test_support.rs @@ -2,7 +2,124 @@ use alloc::vec::Vec; -use super::Sequence; +use super::{CompressionLevel, HistoryBuf, MatchGeneratorDriver, Matcher, Sequence}; + +/// The built-in driver reached only through [`Matcher`], as a caller wrapping +/// it does: the encoder sees no built-in driver, so every block runs through +/// the trait entry points under the driver's own tier. +pub(crate) struct TraitDriver(pub(crate) MatchGeneratorDriver); + +impl TraitDriver { + /// A driver whose trait entry points run under `tier`. + pub(crate) fn new(tier: super::fastpath::FastpathKernel) -> Self { + let mut driver = MatchGeneratorDriver::new(crate::common::MAX_BLOCK_SIZE as usize, 1); + driver.set_tier(tier); + Self(driver) + } +} + +impl Matcher for TraitDriver { + fn get_last_space(&mut self) -> &[u8] { + self.0.get_last_space() + } + + fn fill_in_place( + &mut self, + capacity: usize, + fill: &mut dyn FnMut(&mut HistoryBuf) -> (usize, bool), + ) -> (usize, bool) { + self.0.fill_in_place(capacity, fill) + } + + fn uncommitted_input(&self) -> &[u8] { + self.0.uncommitted_input() + } + + fn commit_filled(&mut self, len: usize) { + self.0.commit_filled(len); + } + + fn skip_matching(&mut self) { + self.0.skip_matching(); + } + + fn skip_matching_with_hint(&mut self, incompressible_hint: Option) { + self.0.skip_matching_with_hint(incompressible_hint); + } + + fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { + self.0.start_matching(handle_sequence); + } + + fn reset(&mut self, level: CompressionLevel) { + self.0.reset(level); + } + + fn set_source_size_hint(&mut self, size: u64) { + self.0.set_source_size_hint(size); + } + + fn set_dictionary_size_hint(&mut self, sizes: super::DictionarySizes) { + self.0.set_dictionary_size_hint(sizes); + } + + fn prime_with_dictionary(&mut self, dict_content: &[u8], offset_hist: [u32; 3]) { + self.0.prime_with_dictionary(dict_content, offset_hist); + } + + fn seed_dictionary_entropy( + &mut self, + huff: Option<&crate::huff0::huff0_encoder::HuffmanTable>, + ll: Option<&crate::fse::fse_encoder::FSETable>, + ml: Option<&crate::fse::fse_encoder::FSETable>, + of: Option<&crate::fse::fse_encoder::FSETable>, + ) { + self.0.seed_dictionary_entropy(huff, ll, ml, of); + } + + fn supports_dictionary_priming(&self) -> bool { + self.0.supports_dictionary_priming() + } + + fn settle_block_cut(&mut self, window: usize, cut: usize) -> usize { + self.0.settle_block_cut(window, cut) + } + + fn window_size(&self) -> u64 { + self.0.window_size() + } +} + +/// Levels across every backend: the negative and positive Fast band, dfast, +/// the row lazy band, btlazy2 and the optimal parsers. +pub(crate) const TIER_LEVELS: [i32; 7] = [-5, 1, 3, 6, 13, 16, 19]; + +/// Input that takes each block path: full blocks of log lines, so the frame +/// saves enough for the pre-splitter, then a block that turns to noise halfway +/// (a pre-split cut), a block of one byte repeated (a block written without +/// a search) and a short tail of text. +pub(crate) fn tier_fixture() -> Vec { + const BLOCK: usize = crate::common::MAX_BLOCK_SIZE as usize; + let line = b"ts=2026-03-26T21:39:28Z level=INFO msg=\"flush memtable\" table=orders\n"; + let mut data = line.repeat((2 * BLOCK + BLOCK / 2) / line.len() + 1); + data.truncate(2 * BLOCK + BLOCK / 2); + let mut state = 0x2545_F491_u32; + data.extend((0..BLOCK / 2).map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + })); + data.extend(core::iter::repeat_n(b'z', BLOCK)); + data.extend_from_slice(&line.repeat(40)); + data +} + +/// A raw-content dictionary of the fixture's log lines. +pub(crate) fn tier_dictionary() -> crate::decoding::Dictionary { + let content = b"level=INFO msg=\"rotate segment\" table=orders region=eu-west\n".repeat(64); + crate::decoding::Dictionary::from_raw_content(0xD1C7_7135, content).unwrap() +} /// Rebuilds a block from its sequences the way a decoder does: each literal /// run is taken from the block at the position the sequences before it reach, diff --git a/zstd/src/huff0/huf_cstream.rs b/zstd/src/huff0/huf_cstream.rs index 199826abd..a8a6acf74 100644 --- a/zstd/src/huff0/huf_cstream.rs +++ b/zstd/src/huff0/huf_cstream.rs @@ -25,6 +25,8 @@ use alloc::vec::Vec; +use crate::encoding::fastpath::EncodeTier; + /// Upstream zstd `HUF_BITS_IN_CONTAINER = sizeof(size_t) * 8`. We hard-code 64 /// regardless of target pointer width. Upstream zstd's `MEM_32bits()` branch /// switches the container to `u32` on 32-bit hosts; this crate's CI @@ -111,24 +113,6 @@ pub(crate) struct HufCStream<'a> { overflow: bool, } -/// Whether to run the BMI2-targeted burst-encode kernel. BMI2 lets the codegen -/// use `shrx`/`shlx` for the bit-container's variable shifts (no CL dependency, -/// no flag stall) instead of `shr`/`shl`; the emitted instruction stream differs -/// but the logic — and therefore the compressed output — is identical. Detected -/// at the per-stream entry, never inside the symbol loop. -#[cfg(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64")))] -#[inline] -fn huf_encode_use_bmi2() -> bool { - use crate::cpu_kernel::{CpuLevel, cpu_allows}; - cpu_allows(CpuLevel::Bmi2) && std::arch::is_x86_feature_detected!("bmi2") -} -#[cfg(all(not(feature = "std"), any(target_arch = "x86", target_arch = "x86_64")))] -#[inline] -fn huf_encode_use_bmi2() -> bool { - use crate::cpu_kernel::{CpuLevel, cpu_allows}; - cpu_allows(CpuLevel::Bmi2) && cfg!(target_feature = "bmi2") -} - /// One symbol into bit container `$bc` / bit position `$bp`. Mirrors /// `add_bits`; `$fast` (a `const`-valued bool) const-folds. Module-level so the /// burst-loop body macro can expand it inside both the scalar and the @@ -367,8 +351,13 @@ impl<'a> HufCStream<'a> { /// `2 * K_UNROLL`, (3) dual-container main loop processing /// `2 * K_UNROLL` symbols per iteration. Symbols consumed in reverse /// (`data[--n]`). + /// + /// The kernel is the encoder tier `K`'s: BMI2 lets the container's variable + /// shifts be `shrx`/`shlx` (no `CL` dependency, no flag stall) instead of + /// `shr`/`shl`. The instructions differ, the output does not. #[inline] pub(crate) fn encode_unrolled< + K: EncodeTier, const K_UNROLL: usize, const K_FAST_FLUSH: bool, const K_LAST_FAST: bool, @@ -377,20 +366,19 @@ impl<'a> HufCStream<'a> { table: &[u64], data: &[u8], ) { - // Pick the burst kernel ONCE per stream, never inside the symbol loop. - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - if huf_encode_use_bmi2() { - // SAFETY: `huf_encode_use_bmi2()` returned true, so the running - // CPU has BMI2; the body adds no unsafe beyond the 8-byte store - // already justified in `huf_flush!`. - unsafe { - self.encode_unrolled_bmi2::(table, data); - } - return; - } - } - self.encode_unrolled_scalar::(table, data); + crate::encoding::fastpath::tier_match!(K { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] + // SAFETY: the tier is only selected after confirming BMI2 (and + // AVX2); the body adds no unsafe beyond the 8-byte store already + // justified in `huf_flush!`. + crate::encoding::fastpath::FastpathKernel::Avx2Bmi2 => unsafe { + self.encode_unrolled_bmi2::(table, data) + }, + _ => self.encode_unrolled_scalar::(table, data), + }) } /// Portable burst-encode kernel (no target feature). Bit-identical output to @@ -412,8 +400,12 @@ impl<'a> HufCStream<'a> { /// the container's variable shifts lower to `shrx`/`shlx`. /// /// # Safety - /// The caller must have verified BMI2 is available (`huf_encode_use_bmi2()`). - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + /// The caller must have verified BMI2 is available: the encoder tier + /// running it is [`crate::encoding::fastpath::FastpathKernel::Avx2Bmi2`]. + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] #[target_feature(enable = "bmi2")] unsafe fn encode_unrolled_bmi2< const K_UNROLL: usize, diff --git a/zstd/src/huff0/huf_cstream/tests.rs b/zstd/src/huff0/huf_cstream/tests.rs index e180a72da..1458c19dd 100644 --- a/zstd/src/huff0/huf_cstream/tests.rs +++ b/zstd/src/huff0/huf_cstream/tests.rs @@ -66,7 +66,7 @@ fn encode_unrolled_dual_container_size_is_deterministic() { // Every symbol maps to the same 4-bit code (value 0b1010). let table = [pack_huf_celt(0b1010, 4); 256]; let data = [0u8; 16]; - s.encode_unrolled::<4, false, false>(&table, &data); + s.encode_unrolled::(&table, &data); let n = s.close(); assert_eq!( n, 9, diff --git a/zstd/src/huff0/huff0_encoder.rs b/zstd/src/huff0/huff0_encoder.rs index 260e62e86..64a07d4fd 100644 --- a/zstd/src/huff0/huff0_encoder.rs +++ b/zstd/src/huff0/huff0_encoder.rs @@ -100,6 +100,7 @@ impl Clone for WeightDescription { use crate::{ bit_io::BitWriter, + encoding::fastpath::EncodeTier, fse::fse_encoder::{self, FSEEncoder}, histogram, }; @@ -155,7 +156,9 @@ impl>> HuffmanEncoder<'_, '_, V> { /// (`huf_compress.c:1077-1108`) so each instantiation keeps the /// container's per-symbol nb_bits sum below 64 - 4 (the upstream zstd "≥ /// 4 free bits" precondition for `kFast=1`). - pub fn encode4x(&mut self, data: &[u8], with_table: bool) { + /// + /// The streams are written with the burst kernel of the encoder tier `K`. + pub fn encode4x(&mut self, data: &[u8], with_table: bool) { assert!( data.len() >= 12, "upstream zstd HUF_compress4X requires srcSize >= 12" @@ -203,7 +206,7 @@ impl>> HuffmanEncoder<'_, '_, V> { "HufCStream::new returned None — dst_capacity (from \ huf_tight_compress_bound) must be > 8 for a non-empty segment", ); - Self::encode_one_stream(packed_codes, &mut bit_c, segment, table_log); + Self::encode_one_stream::(packed_codes, &mut bit_c, segment, table_log); bit_c.close() }); // Upstream zstd `HUF_CStream_close` returns 0 on overflow (the @@ -249,7 +252,7 @@ impl>> HuffmanEncoder<'_, '_, V> { /// Dispatch on table_log to pick `kUnroll`, `kFastFlush`, /// `kLastFast` matching upstream zstd's 64-bit branch in /// `huf_compress.c:1092-1110`. - fn encode_one_stream( + fn encode_one_stream( table: &[u64], bit_c: &mut super::huf_cstream::HufCStream<'_>, data: &[u8], @@ -262,14 +265,14 @@ impl>> HuffmanEncoder<'_, '_, V> { // routes here, which clevels.h does not produce for L1-L22 // Fast/DFast workloads but is correct to handle defensively. match table_log { - 11 => Self::encode_one_stream_unrolled::<5, true, false>(table, bit_c, data), - 10 => Self::encode_one_stream_unrolled::<5, true, true>(table, bit_c, data), - 9 => Self::encode_one_stream_unrolled::<6, true, false>(table, bit_c, data), - 8 => Self::encode_one_stream_unrolled::<7, true, false>(table, bit_c, data), - 7 => Self::encode_one_stream_unrolled::<8, true, false>(table, bit_c, data), + 11 => Self::encode_one_stream_unrolled::(table, bit_c, data), + 10 => Self::encode_one_stream_unrolled::(table, bit_c, data), + 9 => Self::encode_one_stream_unrolled::(table, bit_c, data), + 8 => Self::encode_one_stream_unrolled::(table, bit_c, data), + 7 => Self::encode_one_stream_unrolled::(table, bit_c, data), // tableLog ∈ {1..=6, 12} — upstream zstd's "default" branch falls // here too. kUnroll=4 keeps per-flush bit budget safe. - _ => Self::encode_one_stream_unrolled::<4, false, false>(table, bit_c, data), + _ => Self::encode_one_stream_unrolled::(table, bit_c, data), } } @@ -300,6 +303,7 @@ impl>> HuffmanEncoder<'_, '_, V> { /// each unroll group. Safe when `K_UNROLL * max_nb_bits + 4 <= /// 64`, i.e. `K_UNROLL <= (60 / table_log)`. fn encode_one_stream_unrolled< + K: EncodeTier, const K_UNROLL: usize, const K_FAST_FLUSH: bool, const K_LAST_FAST: bool, @@ -312,7 +316,7 @@ impl>> HuffmanEncoder<'_, '_, V> { // which keeps the bit containers / positions / cursor // register-resident (upstream zstd's `HUF_CStream_t` shape) instead of // reloading them from `*bit_c` per symbol. Byte-identical output. - bit_c.encode_unrolled::(table, data); + bit_c.encode_unrolled::(table, data); } /// Encode one stream and pad it to fill the last byte @@ -2035,7 +2039,10 @@ pub(crate) fn huf_encode4x_for_test(data: &[u8]) -> Vec { { let mut writer = BitWriter::from(&mut encoded); let mut encoder = HuffmanEncoder::new(&table, &mut writer); - encoder.encode4x(data, true); + crate::encoding::fastpath::with_encode_tier!( + crate::encoding::fastpath::select_kernel(), + K => encoder.encode4x::(data, true) + ); writer.flush(); } encoded