From 48fb43165b1ffab65657cba3240afbcfe806a1be Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sat, 5 Sep 2026 19:40:52 +0000 Subject: [PATCH 1/5] dfast: pick cmov vs. branch for candidate checks by table freshness The fast match finder tests whether a table candidate lies inside the window with either a conditional move or a branch, chosen per block. The branch wins when nearly every candidate is in range; the conditional move wins when many entries are stale and the branch would mispredict. The double-fast noDict loop always used the conditional move, which is the slower form on x86-64 at level 3 on large inputs. Share the helpers between the two match finders (adding 8-byte variants for the long table) and let double-fast choose too, by table freshness rather than window size alone. A slot goes stale only if it is not overwritten for a whole window, which is rare once windowLog >= hashLog + 4 and common when the tables are about as large as the window. So level 3 on large inputs takes the branch; level 4 on large inputs and any small window keep the conditional move. The long candidate at ip+1 goes through the same helper instead of a plain branch. zstd -b on silesia.tar and enwik8: level 3 is 1-4% faster on x86-64 and unchanged on aarch64, where GCC already emitted a branch; level 4 is unchanged; levels 1-2 are not touched. One-shot output is byte-identical. Streaming output can differ by a few bytes once the window slides (at most 0.006%, on silesia and on a synthetic stream of small messages), because the ip+1 long candidate is now accepted at exactly the lowest valid index, as the ip candidates already were. On aarch64 with GCC this change alone costs 1.7-2.5% where the conditional move is used, because GCC turns the ternary in ZSTD_selectAddr() (and so the new ip+1 select) back into a branch. The next commit adds an aarch64 csel there to recover that. Co-authored-by: Claude --- lib/compress/zstd_compress_internal.h | 76 ++++++++++++++ lib/compress/zstd_double_fast.c | 146 +++++++++++++++----------- lib/compress/zstd_fast.c | 45 +------- 3 files changed, 164 insertions(+), 103 deletions(-) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index e233bc5ff1b..be4e708c5b6 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -665,6 +665,82 @@ ZSTD_selectAddr(U32 index, U32 lowLimit, const BYTE* candidate, const BYTE* back #endif } +/* ZSTD_match4Found_* / ZSTD_match8Found_* : + * @return 1 when the 4 (resp. 8) bytes at matchAddress equal those at currentPtr, + * for a hash-table candidate that may only be dereferenced when matchIdx >= idxLowLimit. + * The _cmov variants never branch on index validity: an out-of-range candidate is + * redirected to a dummy buffer, so the only branch left is on the byte comparison, + * which is rarely taken and predicts well. They are the right choice when validity + * is unpredictable, i.e. when the window is not much larger than the tables, so + * that many entries are stale, and in extDict mode where the valid range moves + * with every block. + * The _branch variants test validity first; they are faster when nearly every + * candidate is within range (large windows), where that branch is free and the + * select would only lengthen the dependency chain in front of the candidate load. */ +typedef int (*ZSTD_matchFound)(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit); + +MEM_STATIC int +ZSTD_match4Found_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) +{ + /* Array of ~random data, should have low probability of matching data. + * Load from here if the index is invalid. + * Used to avoid unpredictable branches. */ + static const BYTE dummy[] = {0x12,0x34,0x56,0x78}; + + const BYTE* mvalAddr = ZSTD_selectAddr(matchIdx, idxLowLimit, matchAddress, dummy); + /* Note: this used to be written as : return test1 && test2; + * Unfortunately, once inlined, these tests become branches, + * in which case it becomes critical that they are executed in the right order (test1 then test2). + * So we have to write these tests in a specific manner to ensure their ordering. + */ + if (MEM_read32(currentPtr) != MEM_read32(mvalAddr)) return 0; + /* force ordering of these tests, which matters once the function is inlined, as they become branches */ +#if defined(__GNUC__) + __asm__(""); +#endif + return matchIdx >= idxLowLimit; +} + +MEM_STATIC int +ZSTD_match4Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) +{ + /* using a branch instead of a cmov, + * because it's faster in scenarios where matchIdx >= idxLowLimit is generally true, + * aka almost all candidates are within range */ + U32 mval; + if (matchIdx >= idxLowLimit) { + mval = MEM_read32(matchAddress); + } else { + mval = MEM_read32(currentPtr) ^ 1; /* guaranteed to not match. */ + } + + return (MEM_read32(currentPtr) == mval); +} + +MEM_STATIC int +ZSTD_match8Found_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) +{ + static const BYTE dummy[] = {0x12,0x34,0x56,0x78,0x9a,0xbc,0xde,0xf0}; + const BYTE* mvalAddr = ZSTD_selectAddr(matchIdx, idxLowLimit, matchAddress, dummy); + if (MEM_read64(currentPtr) != MEM_read64(mvalAddr)) return 0; +#if defined(__GNUC__) + __asm__(""); +#endif + return matchIdx >= idxLowLimit; +} + +MEM_STATIC int +ZSTD_match8Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) +{ + U64 mval; + if (matchIdx >= idxLowLimit) { + mval = MEM_read64(matchAddress); + } else { + mval = MEM_read64(currentPtr) ^ 1; /* guaranteed to not match. */ + } + return (MEM_read64(currentPtr) == mval); +} + /* ZSTD_noCompressBlock() : * Writes uncompressed block to dst buffer from given src. * Returns the size of the block */ diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c index 1a266e7d955..168bb2a55bc 100644 --- a/lib/compress/zstd_double_fast.c +++ b/lib/compress/zstd_double_fast.c @@ -13,6 +13,7 @@ #ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + static ZSTD_ALLOW_POINTER_OVERFLOW_ATTR void ZSTD_fillDoubleHashTableForCDict(ZSTD_MatchState_t* ms, @@ -104,7 +105,8 @@ FORCE_INLINE_TEMPLATE ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_compressBlock_doubleFast_noDict_generic( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize, U32 const mls /* template */) + void const* src, size_t srcSize, + U32 const mls /* template */, int const useCmov /* template */) { ZSTD_compressionParameters const* cParams = &ms->cParams; U32* const hashLong = ms->hashTable; @@ -142,14 +144,12 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( const BYTE* matchl0; /* the long match for ip */ const BYTE* matchs0; /* the short match for ip */ const BYTE* matchl1; /* the long match for ip1 */ - const BYTE* matchs0_safe; /* matchs0 or safe address */ const BYTE* ip = istart; /* the current position */ const BYTE* ip1; /* the next position */ - /* Array of ~random data, should have low probability of matching data - * we load from here instead of from tables, if matchl0/matchl1 are - * invalid indices. Used to avoid unpredictable branches. */ - const BYTE dummy[] = {0x12,0x34,0x56,0x78,0x9a,0xbc,0xde,0xf0,0xe2,0xb4}; + + const ZSTD_matchFound matchLongFound = useCmov ? ZSTD_match8Found_cmov : ZSTD_match8Found_branch; + const ZSTD_matchFound matchShortFound = useCmov ? ZSTD_match4Found_cmov : ZSTD_match4Found_branch; DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic"); @@ -196,29 +196,20 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( hl1 = ZSTD_hashPtr(ip1, hBitsL, 8); - /* idxl0 > prefixLowestIndex is a (somewhat) unpredictable branch. - * However expression below complies into conditional move. Since - * match is unlikely and we only *branch* on idxl0 > prefixLowestIndex - * if there is a match, all branches become predictable. */ - { const BYTE* const matchl0_safe = ZSTD_selectAddr(idxl0, prefixLowestIndex, matchl0, &dummy[0]); - - /* check prefix long match */ - if (MEM_read64(matchl0_safe) == MEM_read64(ip) && matchl0_safe == matchl0) { - mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8; - offset = (U32)(ip-matchl0); - while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ - goto _match_found; - } } + /* check prefix long match */ + if (matchLongFound(ip, matchl0, idxl0, prefixLowestIndex)) { + mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8; + offset = (U32)(ip-matchl0); + while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ + goto _match_found; + } idxl1 = hashLong[hl1]; matchl1 = base + idxl1; - /* Same optimization as matchl0 above */ - matchs0_safe = ZSTD_selectAddr(idxs0, prefixLowestIndex, matchs0, &dummy[0]); - /* check prefix short match */ - if(MEM_read32(matchs0_safe) == MEM_read32(ip) && matchs0_safe == matchs0) { - goto _search_next_long; + if (matchShortFound(ip, matchs0, idxs0, prefixLowestIndex)) { + goto _search_next_long; } if (ip1 >= nextStep) { @@ -257,7 +248,7 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( offset = (U32)(ip - matchs0); /* check long match at +1 position */ - if ((idxl1 > prefixLowestIndex) && (MEM_read64(matchl1) == MEM_read64(ip1))) { + if (matchLongFound(ip1, matchl1, idxl1, prefixLowestIndex)) { size_t const l1len = ZSTD_count(ip1+8, matchl1+8, iend) + 8; if (l1len > mLength) { /* use the long match instead */ @@ -328,7 +319,7 @@ ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize, - U32 const mls /* template */) + U32 const mls /* template */, int const useCmov) { ZSTD_compressionParameters const* cParams = &ms->cParams; U32* const hashLong = ms->hashTable; @@ -361,6 +352,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic"); + (void)useCmov; /* if a dictionary is attached, it must be within window range */ assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); @@ -546,23 +538,28 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( return (size_t)(iend - anchor); } -#define ZSTD_GEN_DFAST_FN(dictMode, mls) \ - static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ - void const* src, size_t srcSize) \ - { \ - return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \ +#define ZSTD_GEN_DFAST_FN(dictMode, mls, cmov) \ + static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls##_##cmov( \ + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls, cmov); \ } -ZSTD_GEN_DFAST_FN(noDict, 4) -ZSTD_GEN_DFAST_FN(noDict, 5) -ZSTD_GEN_DFAST_FN(noDict, 6) -ZSTD_GEN_DFAST_FN(noDict, 7) +ZSTD_GEN_DFAST_FN(noDict, 4, 1) +ZSTD_GEN_DFAST_FN(noDict, 5, 1) +ZSTD_GEN_DFAST_FN(noDict, 6, 1) +ZSTD_GEN_DFAST_FN(noDict, 7, 1) + +ZSTD_GEN_DFAST_FN(noDict, 4, 0) +ZSTD_GEN_DFAST_FN(noDict, 5, 0) +ZSTD_GEN_DFAST_FN(noDict, 6, 0) +ZSTD_GEN_DFAST_FN(noDict, 7, 0) -ZSTD_GEN_DFAST_FN(dictMatchState, 4) -ZSTD_GEN_DFAST_FN(dictMatchState, 5) -ZSTD_GEN_DFAST_FN(dictMatchState, 6) -ZSTD_GEN_DFAST_FN(dictMatchState, 7) +ZSTD_GEN_DFAST_FN(dictMatchState, 4, 0) +ZSTD_GEN_DFAST_FN(dictMatchState, 5, 0) +ZSTD_GEN_DFAST_FN(dictMatchState, 6, 0) +ZSTD_GEN_DFAST_FN(dictMatchState, 7, 0) size_t ZSTD_compressBlock_doubleFast( @@ -570,17 +567,40 @@ size_t ZSTD_compressBlock_doubleFast( void const* src, size_t srcSize) { const U32 mls = ms->cParams.minMatch; - switch(mls) - { - default: /* includes case 3 */ - case 4 : - return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); - case 5 : - return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); - case 6 : - return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); - case 7 : - return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); + /* A table entry goes stale (falls out of the window) when its slot is not + * overwritten for a whole window span. With about one insertion per position + * that is rare once the window is much larger than the tables, and the + * "candidate in range" test then predicts perfectly: use a branch. + * Otherwise (small windows, or tables sized close to the window) many + * entries are stale and the test is unpredictable: use a cmov. */ + int const useCmov = ms->cParams.windowLog < ms->cParams.hashLog + 4; + if (useCmov) { + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_noDict_4_1(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_noDict_5_1(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_noDict_6_1(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_noDict_7_1(ms, seqStore, rep, src, srcSize); + } + } else { + /* use a branch instead */ + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_noDict_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_noDict_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_noDict_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_noDict_7_0(ms, seqStore, rep, src, srcSize); + } } } @@ -594,13 +614,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); + return ZSTD_compressBlock_doubleFast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); + return ZSTD_compressBlock_doubleFast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); + return ZSTD_compressBlock_doubleFast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); + return ZSTD_compressBlock_doubleFast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize); } } @@ -751,10 +771,18 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( return (size_t)(iend - anchor); } -ZSTD_GEN_DFAST_FN(extDict, 4) -ZSTD_GEN_DFAST_FN(extDict, 5) -ZSTD_GEN_DFAST_FN(extDict, 6) -ZSTD_GEN_DFAST_FN(extDict, 7) +#define ZSTD_GEN_DFAST_EXTDICT_FN(mls) \ + static size_t ZSTD_compressBlock_doubleFast_extDict_##mls( \ + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ + } + +ZSTD_GEN_DFAST_EXTDICT_FN(4) +ZSTD_GEN_DFAST_EXTDICT_FN(5) +ZSTD_GEN_DFAST_EXTDICT_FN(6) +ZSTD_GEN_DFAST_EXTDICT_FN(7) size_t ZSTD_compressBlock_doubleFast_extDict( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c index ee25bcbac8d..178575aa222 100644 --- a/lib/compress/zstd_fast.c +++ b/lib/compress/zstd_fast.c @@ -97,49 +97,6 @@ void ZSTD_fillHashTable(ZSTD_MatchState_t* ms, } -typedef int (*ZSTD_match4Found) (const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit); - -static int -ZSTD_match4Found_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) -{ - /* Array of ~random data, should have low probability of matching data. - * Load from here if the index is invalid. - * Used to avoid unpredictable branches. */ - static const BYTE dummy[] = {0x12,0x34,0x56,0x78}; - - /* currentIdx >= lowLimit is a (somewhat) unpredictable branch. - * However expression below compiles into conditional move. - */ - const BYTE* mvalAddr = ZSTD_selectAddr(matchIdx, idxLowLimit, matchAddress, dummy); - /* Note: this used to be written as : return test1 && test2; - * Unfortunately, once inlined, these tests become branches, - * in which case it becomes critical that they are executed in the right order (test1 then test2). - * So we have to write these tests in a specific manner to ensure their ordering. - */ - if (MEM_read32(currentPtr) != MEM_read32(mvalAddr)) return 0; - /* force ordering of these tests, which matters once the function is inlined, as they become branches */ -#if defined(__GNUC__) - __asm__(""); -#endif - return matchIdx >= idxLowLimit; -} - -static int -ZSTD_match4Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) -{ - /* using a branch instead of a cmov, - * because it's faster in scenarios where matchIdx >= idxLowLimit is generally true, - * aka almost all candidates are within range */ - U32 mval; - if (matchIdx >= idxLowLimit) { - mval = MEM_read32(matchAddress); - } else { - mval = MEM_read32(currentPtr) ^ 1; /* guaranteed to not match. */ - } - - return (MEM_read32(currentPtr) == mval); -} - /** * If you squint hard enough (and ignore repcodes), the search operation at any @@ -232,7 +189,7 @@ size_t ZSTD_compressBlock_fast_noDict_generic( size_t step; const BYTE* nextStep; const size_t kStepIncr = (1 << (kSearchStrength - 1)); - const ZSTD_match4Found matchFound = useCmov ? ZSTD_match4Found_cmov : ZSTD_match4Found_branch; + const ZSTD_matchFound matchFound = useCmov ? ZSTD_match4Found_cmov : ZSTD_match4Found_branch; DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); ip0 += (ip0 == prefixStart); From d69bd43719d3657efc5a913a49c7d21f28eb771b Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sat, 5 Sep 2026 19:40:52 +0000 Subject: [PATCH 2/5] aarch64: make ZSTD_selectAddr() branchless ZSTD_selectAddr() exists so that the "is this table index inside the window" test in the fast and double-fast candidate checks compiles to a conditional move, not a branch that mispredicts when many entries are stale, as with a small window. x86-64 forces this with inline asm; elsewhere it is a plain ternary, which GCC 13 and 15 on aarch64 turn back into a branch, though clang does not. Force it on aarch64 too, with an inline-asm csel. Output is unchanged. Neoverse-V2 (Graviton4), GCC 13: double-fast on synthetic small records with a 32 KB window and level-3 tables (wlog=15, hlog=14, clog=13) is 10% faster. zstd -b on silesia.tar and enwik8: level 4 is 2-3% faster, levels 1-2 unchanged; level 3, where nearly all candidates are in range, is unchanged now that double-fast branches when windowLog >= hashLog + 4 (it would be 2.5% slower if double-fast still used the select there). Co-authored-by: Claude --- lib/compress/zstd_compress_internal.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index be4e708c5b6..7bd27286468 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -660,6 +660,14 @@ ZSTD_selectAddr(U32 index, U32 lowLimit, const BYTE* candidate, const BYTE* back : "r"(index), "r"(lowLimit), "r"(backup) ); return candidate; +#elif defined(__GNUC__) && defined(__aarch64__) + __asm__ ( + "cmp %w1, %w2\n" + "csel %0, %3, %0, lo\n" + : "+r"(candidate) + : "r"(index), "r"(lowLimit), "r"(backup) + : "cc"); + return candidate; #else return index >= lowLimit ? candidate : backup; #endif From e3632782b7cfa98e7c96119691f37c4dabc40157 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sat, 5 Sep 2026 19:40:52 +0000 Subject: [PATCH 3/5] compress: force-inlined ZSTD_count variants for zstd_double_fast.c ZSTD_count() and ZSTD_count_2segments() are a handful of instructions on the hottest path of every block compressor and are meant to be inlined. GCC inlines them with default flags. With the hardening flags distributions commonly add (-D_FORTIFY_SOURCE=3, -fstack-protector-strong, -fstack-clash-protection), GCC 13-15 at -O3 stop inlining them in several zstd_double_fast.c specializations once the file gains a few more specializations, as the next commit does; the calls then cost about 2% on match-dense inputs. Add ZSTD_count_inline() and ZSTD_count_2segments_inline(), the same code marked FORCE_INLINE_TEMPLATE, and use them throughout zstd_double_fast.c. The original functions are untouched, so the other block compressors compile to the same code as before (function sizes in zstd_fast.o, zstd_lazy.o, zstd_opt.o and zstd_ldm.o verified identical). Compressed output does not change. Co-authored-by: Claude --- lib/compress/zstd_compress_internal.h | 39 +++++++++++++++++++++++++++ lib/compress/zstd_double_fast.c | 36 ++++++++++++------------- 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 7bd27286468..3fe9ad120bf 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -997,6 +997,45 @@ ZSTD_count_2segments(const BYTE* ip, const BYTE* match, return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd); } +/* ZSTD_count_inline(), ZSTD_count_2segments_inline() : + * same results as ZSTD_count() / ZSTD_count_2segments(), with inlining forced. + * For block compressors whose translation unit carries enough template + * specializations that the compiler's unit-growth limits would otherwise turn + * these few hot instructions into calls (seen with GCC 13-15 at -O3 on + * zstd_double_fast.c). Kept separate so that code generation for every other + * caller of ZSTD_count() is unchanged. */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_count_inline(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) +{ + const BYTE* const pStart = pIn; + const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1); + + if (pIn < pInLoopLimit) { + { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (diff) return ZSTD_NbCommonBytes(diff); } + pIn+=sizeof(size_t); pMatch+=sizeof(size_t); + while (pIn < pInLoopLimit) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } + pIn += ZSTD_NbCommonBytes(diff); + return (size_t)(pIn - pStart); + } } + if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { - mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + mLength = ZSTD_count_inline(ip+1+4, ip+1+4-offset_1, iend) + 4; ip++; ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); goto _match_stored; @@ -198,7 +198,7 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( /* check prefix long match */ if (matchLongFound(ip, matchl0, idxl0, prefixLowestIndex)) { - mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8; + mLength = ZSTD_count_inline(ip+8, matchl0+8, iend) + 8; offset = (U32)(ip-matchl0); while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ goto _match_found; @@ -244,12 +244,12 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( _search_next_long: /* short match found: let's check for a longer one */ - mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4; + mLength = ZSTD_count_inline(ip+4, matchs0+4, iend) + 4; offset = (U32)(ip - matchs0); /* check long match at +1 position */ if (matchLongFound(ip1, matchl1, idxl1, prefixLowestIndex)) { - size_t const l1len = ZSTD_count(ip1+8, matchl1+8, iend) + 8; + size_t const l1len = ZSTD_count_inline(ip1+8, matchl1+8, iend) + 8; if (l1len > mLength) { /* use the long match instead */ ip = ip1; @@ -300,7 +300,7 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( && ( (offset_2>0) & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { /* store sequence */ - size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + size_t const rLength = ZSTD_count_inline(ip+4, ip+4-offset_2, iend) + 4; U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); @@ -399,7 +399,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex)) && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + mLength = ZSTD_count_2segments_inline(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; ip++; ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); goto _match_stored; @@ -407,7 +407,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( if ((matchIndexL >= prefixLowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { /* check prefix long match */ - mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; + mLength = ZSTD_count_inline(ip+8, matchLong+8, iend) + 8; offset = (U32)(ip-matchLong); while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ goto _match_found; @@ -418,7 +418,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( assert(dictMatchL < dictEnd); if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) { - mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8; + mLength = ZSTD_count_2segments_inline(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8; offset = (U32)(curr - dictMatchIndexL - dictIndexDelta); while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ goto _match_found; @@ -456,7 +456,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( /* check prefix long +1 match */ if ((matchIndexL3 >= prefixLowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1))) { - mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; + mLength = ZSTD_count_inline(ip+9, matchL3+8, iend) + 8; ip++; offset = (U32)(ip-matchL3); while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ @@ -467,7 +467,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3; assert(dictMatchL3 < dictEnd); if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) { - mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8; + mLength = ZSTD_count_2segments_inline(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8; ip++; offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta); while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ @@ -476,11 +476,11 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( /* if no long +1 match, explore the short match we found */ if (matchIndexS < prefixLowestIndex) { - mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; + mLength = ZSTD_count_2segments_inline(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; offset = (U32)(curr - matchIndexS); while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ } else { - mLength = ZSTD_count(ip+4, match+4, iend) + 4; + mLength = ZSTD_count_inline(ip+4, match+4, iend) + 4; offset = (U32)(ip - match); while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ } @@ -516,7 +516,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex2)) && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; + size_t const repLength2 = ZSTD_count_2segments_inline(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; @@ -683,7 +683,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( & (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */ && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; + mLength = ZSTD_count_2segments_inline(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); } else { @@ -691,7 +691,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart; U32 offset; - mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8; + mLength = ZSTD_count_2segments_inline(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8; offset = curr - matchLongIndex; while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ offset_2 = offset_1; @@ -708,14 +708,14 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend; const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart; - mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8; + mLength = ZSTD_count_2segments_inline(ip+9, match3+8, iend, matchEnd, prefixStart) + 8; ip++; offset = curr+1 - matchIndex3; while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ } else { const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; - mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; + mLength = ZSTD_count_2segments_inline(ip+4, match+4, iend, matchEnd, prefixStart) + 4; offset = curr - matchIndex; while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ } @@ -751,7 +751,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( & (offset_2 <= current2 - dictStartIndex)) && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + size_t const repLength2 = ZSTD_count_2segments_inline(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; From b68b30896556506f0ccce7e91942d691b8629fbe Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sat, 5 Sep 2026 19:40:52 +0000 Subject: [PATCH 4/5] dfast: pipelined extDict block compressor When a stream with a small window is flushed every few KB, zstd's input buffer wraps about once per window's worth of input, so nearly every block is compressed in extDict mode, and at level 3 the double-fast extDict loop sets streaming speed. That loop still searched one position at a time; the double-fast noDict and fast extDict loops had already been restructured to overlap work across positions and avoid branching on index validity. Give that loop the structure of the noDict one. The long-table lookup for ip+1 is issued before the candidates at ip are resolved. Candidates are tested with the conditional-move helpers: in extDict mode the lowest valid index moves with every block, so the tables always hold stale entries, and the branch form measured slower or equal at every window size tried (windowLog 15 to 21). Repcodes that cannot be valid anywhere in the block are disabled up front rather than range-checked at every position. Step/skip acceleration, prefetch placement and per-minMatch specialization follow the noDict loop. Segment handling is unchanged: an index below prefixStartIndex is addressed through dictBase, match lengths use the two-segment count, and backward extension stops at the start of the candidate's segment. Output changes slightly with the parse: -0.15% bytes on silesia cut into 4 KB flushes with a 32 KB window, +0.04% on a synthetic message stream. Compression speed of this commit plus the three preceding ones versus unpatched dev; level 3, 32 KB window, one core, GCC 15 (silesia row GCC 13): workload Neoverse-V2 Sapphire Rapids 4.5-8.9 KB flushed messages, 2,048 live streams interleaved +10-11% +7-10% one stream +11% +15% silesia.tar, 4 KB flushes, 16 streams +22% +24% One-shot zstd -b, same four changes, GCC 13: levels 1-2 unchanged, level 3 up to 3% faster, level 4 is 5-6% faster on Neoverse-V2 and unchanged on Sapphire Rapids. The messages are synthetic small structured records, not a public corpus; Neoverse-V2 is an AWS Graviton4. Co-authored-by: Claude --- lib/compress/zstd_double_fast.c | 262 ++++++++++++++++++++++---------- 1 file changed, 185 insertions(+), 77 deletions(-) diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c index 57502f3b6af..77ecb0353d8 100644 --- a/lib/compress/zstd_double_fast.c +++ b/lib/compress/zstd_double_fast.c @@ -625,7 +625,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState( } -static +FORCE_INLINE_TEMPLATE ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_compressBlock_doubleFast_extDict_generic( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], @@ -638,7 +638,6 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( U32* const hashSmall = ms->chainTable; U32 const hBitsS = cParams->chainLog; const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; const BYTE* anchor = istart; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; @@ -653,6 +652,39 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* const dictStart = dictBase + dictStartIndex; const BYTE* const dictEnd = dictBase + prefixStartIndex; U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + size_t mLength; + U32 offset; + U32 curr; + + /* how many positions to search before increasing step size */ + const size_t kStepIncr = 1 << kSearchStrength; + /* the position at which to increment the step size if no match is found */ + const BYTE* nextStep; + size_t step; /* the current step size */ + + size_t hl0; /* the long hash at ip */ + size_t hl1; /* the long hash at ip1 */ + + U32 idxl0; /* the long match index for ip */ + U32 idxs0; /* the short match index for ip */ + U32 idxl1; /* the long match index for ip1 */ + + const BYTE* matchl0; /* the long match for ip */ + const BYTE* matchs0; /* the short match for ip */ + const BYTE* matchl1; /* the long match for ip1 */ + const BYTE* matchEnd; /* end of the segment containing the match */ + const BYTE* matchLow; /* start of the segment containing the match */ + + const BYTE* ip = istart; /* the current position */ + const BYTE* ip1; /* the next position */ + + /* In extDict mode the lowest valid index moves with every block, so + * the tables always hold a mix of in-range and stale entries: test + * candidates without branching on index validity, whatever the window size. */ + const ZSTD_matchFound matchLongFound = ZSTD_match8Found_cmov; + const ZSTD_matchFound matchShortFound = ZSTD_match4Found_cmov; DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize); @@ -660,75 +692,157 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( if (prefixStartIndex == dictStartIndex) return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ - const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); - const U32 matchIndex = hashSmall[hSmall]; - const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; - const BYTE* match = matchBase + matchIndex; - - const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); - const U32 matchLongIndex = hashLong[hLong]; - const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base; - const BYTE* matchLong = matchLongBase + matchLongIndex; - - const U32 curr = (U32)(ip-base); - const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ - const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - size_t mLength; - hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ + /* A table index is dereferenceable iff it is >= dictStartIndex; indexes + * below prefixStartIndex are addressed through dictBase, the others + * through base. Every position was inserted with at least 8 bytes left + * in its own segment, so an in-range candidate can always be read with + * MEM_read64() without running past dictEnd. */ - if (((ZSTD_index_overlap_check(prefixStartIndex, repIndex)) - & (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */ - && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments_inline(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); - } else { - if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { - const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; - const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart; - U32 offset; - mLength = ZSTD_count_2segments_inline(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8; - offset = curr - matchLongIndex; - while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); - - } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { - size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); - U32 const matchIndex3 = hashLong[h3]; - const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base; - const BYTE* match3 = match3Base + matchIndex3; - U32 offset; - hashLong[h3] = curr + 1; - if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { - const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend; - const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart; - mLength = ZSTD_count_2segments_inline(ip+9, match3+8, iend, matchEnd, prefixStart) + 8; - ip++; - offset = curr+1 - matchIndex3; - while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + { U32 const current = (U32)(ip - base); + U32 const maxRep = current - dictStartIndex; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } + + /* Outer Loop: one iteration per match found and stored */ + while (1) { + step = 1; + nextStep = ip + kStepIncr; + ip1 = ip + step; + + if (ip1 > ilimit) { + goto _cleanup; + } + + hl0 = ZSTD_hashPtr(ip, hBitsL, 8); + idxl0 = hashLong[hl0]; + matchl0 = (idxl0 < prefixStartIndex ? dictBase : base) + idxl0; + + /* Inner Loop: one iteration per search / position */ + do { + const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 repIndex = (U32)(ip-base) + 1 - offset_1; + const BYTE* const repMatch = (repIndex < prefixStartIndex ? dictBase : base) + repIndex; + idxs0 = hashSmall[hs0]; + curr = (U32)(ip-base); + matchs0 = (idxs0 < prefixStartIndex ? dictBase : base) + idxs0; + + hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */ + + /* check repcode at ip+1 */ + if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex)) & (offset_1 > 0)) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments_inline(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + goto _match_stored; + } + + hl1 = ZSTD_hashPtr(ip1, hBitsL, 8); + + /* check long match */ + if (matchLongFound(ip, matchl0, idxl0, dictStartIndex)) { + if (idxl0 < prefixStartIndex) { + matchEnd = dictEnd; + matchLow = dictStart; } else { - const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; - const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; - mLength = ZSTD_count_2segments_inline(ip+4, match+4, iend, matchEnd, prefixStart) + 4; - offset = curr - matchIndex; - while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + matchEnd = iend; + matchLow = prefixStart; } - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + mLength = ZSTD_count_2segments_inline(ip+8, matchl0+8, iend, matchEnd, prefixStart) + 8; + offset = curr - idxl0; + while (((ip>anchor) & (matchl0>matchLow)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ + goto _match_found; + } - } else { - ip += ((ip-anchor) >> kSearchStrength) + 1; - continue; - } } + idxl1 = hashLong[hl1]; + matchl1 = (idxl1 < prefixStartIndex ? dictBase : base) + idxl1; - /* move to next sequence start */ + /* check short match */ + if (matchShortFound(ip, matchs0, idxs0, dictStartIndex)) { + if (idxs0 < prefixStartIndex) { + matchEnd = dictEnd; + matchLow = dictStart; + } else { + matchEnd = iend; + matchLow = prefixStart; + } + goto _search_next_long; + } + + if (ip1 >= nextStep) { + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + step++; + nextStep += kStepIncr; + } + ip = ip1; + ip1 += step; + + hl0 = hl1; + idxl0 = idxl1; + matchl0 = matchl1; + #if defined(__aarch64__) + PREFETCH_L1(ip+256); + #endif + } while (ip1 <= ilimit); + +_cleanup: + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_search_next_long: + + /* short match found: let's check for a longer one */ + mLength = ZSTD_count_2segments_inline(ip+4, matchs0+4, iend, matchEnd, prefixStart) + 4; + offset = curr - idxs0; + + /* check long match at +1 position */ + if (matchLongFound(ip1, matchl1, idxl1, dictStartIndex)) { + const BYTE* const matchl1End = idxl1 < prefixStartIndex ? dictEnd : iend; + size_t const l1len = ZSTD_count_2segments_inline(ip1+8, matchl1+8, iend, matchl1End, prefixStart) + 8; + if (l1len > mLength) { + /* use the long match instead */ + ip = ip1; + mLength = l1len; + offset = (U32)(ip-base) - idxl1; + matchs0 = matchl1; + matchLow = idxl1 < prefixStartIndex ? dictStart : prefixStart; + } + } + + while (((ip>anchor) & (matchs0>matchLow)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* complete backward */ + + /* fall-through */ + +_match_found: /* requires ip, offset, mLength */ + offset_2 = offset_1; + offset_1 = offset; + + if (step < 4) { + /* It is unsafe to write this value back to the hashtable when ip1 is + * greater than or equal to the new ip we will have after we're done + * processing this match. Rather than perform that test directly + * (ip1 >= ip + mLength), which costs speed in practice, we do a simpler + * more predictable test. The minmatch even if we take a short match is + * 4 bytes, so as long as step, the distance between ip and ip1 + * (initially) is less than 4, we know ip1 < new ip. */ + hashLong[hl1] = (U32)(ip1 - base); + } + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + +_match_stored: + /* match found */ ip += mLength; anchor = ip; @@ -746,9 +860,8 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( while (ip <= ilimit) { U32 const current2 = (U32)(ip-base); U32 const repIndex2 = current2 - offset_2; - const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; - if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) - & (offset_2 <= current2 - dictStartIndex)) + const BYTE* const repMatch2 = (repIndex2 < prefixStartIndex ? dictBase : base) + repIndex2; + if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) & (offset_2 > 0)) && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments_inline(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; @@ -761,14 +874,9 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( continue; } break; - } } } - - /* save reps for next block */ - rep[0] = offset_1; - rep[1] = offset_2; - - /* Return the last literals size */ - return (size_t)(iend - anchor); + } + } + } } #define ZSTD_GEN_DFAST_EXTDICT_FN(mls) \ From 7bf1756b9ac4635a6ec3a991dde5c605f2b293b2 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Mon, 28 Sep 2026 18:42:35 +0000 Subject: [PATCH 5/5] tests: update regression results for pipelined dfast extDict The pipelined extDict compressor finds different matches than the old loop, so compressed sizes change at dfast levels (0, 3, 4) in extDict scenarios. 81 rows change, none added or removed. silesia.tar streaming: -0.15% to -0.17% github.tar with dict: +0.87% to +1.13% github with dict load: +0.78% github with dict copy: -0.005% to -0.14% --- tests/regression/results.csv | 162 +++++++++++++++++------------------ 1 file changed, 81 insertions(+), 81 deletions(-) diff --git a/tests/regression/results.csv b/tests/regression/results.csv index c0d4f4ae7ff..5648d75a70a 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -65,13 +65,13 @@ github, level -3 with dict, compress github, level -1, compress cctx, 175468 github, level -1 with dict, compress cctx, 42044 github, level 0, compress cctx, 136331 -github, level 0 with dict, compress cctx, 41534 +github, level 0 with dict, compress cctx, 41475 github, level 1, compress cctx, 142365 github, level 1 with dict, compress cctx, 41715 github, level 3, compress cctx, 136331 -github, level 3 with dict, compress cctx, 41534 +github, level 3 with dict, compress cctx, 41475 github, level 4, compress cctx, 136199 -github, level 4 with dict, compress cctx, 41725 +github, level 4 with dict, compress cctx, 41673 github, level 5, compress cctx, 135121 github, level 5 with dict, compress cctx, 38755 github, level 6, compress cctx, 135122 @@ -194,13 +194,13 @@ github.tar, level -3 with dict, zstdcli, github.tar, level -1, zstdcli, 42680 github.tar, level -1 with dict, zstdcli, 41486 github.tar, level 0, zstdcli, 38888 -github.tar, level 0 with dict, zstdcli, 37999 +github.tar, level 0 with dict, zstdcli, 38384 github.tar, level 1, zstdcli, 39340 github.tar, level 1 with dict, zstdcli, 38230 github.tar, level 3, zstdcli, 38888 -github.tar, level 3 with dict, zstdcli, 37999 +github.tar, level 3 with dict, zstdcli, 38384 github.tar, level 4, zstdcli, 38884 -github.tar, level 4 with dict, zstdcli, 37952 +github.tar, level 4 with dict, zstdcli, 38297 github.tar, level 5, zstdcli, 39655 github.tar, level 5 with dict, zstdcli, 39073 github.tar, level 6, zstdcli, 39286 @@ -306,8 +306,8 @@ github, level 0, advanced github, level 0 with dict, advanced one pass, 41118 github, level 0 with dict dms, advanced one pass, 41118 github, level 0 with dict dds, advanced one pass, 41118 -github, level 0 with dict copy, advanced one pass, 41124 -github, level 0 with dict load, advanced one pass, 41847 +github, level 0 with dict copy, advanced one pass, 41122 +github, level 0 with dict load, advanced one pass, 42173 github, level 1, advanced one pass, 142365 github, level 1 with dict, advanced one pass, 41266 github, level 1 with dict dms, advanced one pass, 41266 @@ -318,13 +318,13 @@ github, level 3, advanced github, level 3 with dict, advanced one pass, 41118 github, level 3 with dict dms, advanced one pass, 41118 github, level 3 with dict dds, advanced one pass, 41118 -github, level 3 with dict copy, advanced one pass, 41124 -github, level 3 with dict load, advanced one pass, 41847 +github, level 3 with dict copy, advanced one pass, 41122 +github, level 3 with dict load, advanced one pass, 42173 github, level 4, advanced one pass, 136199 github, level 4 with dict, advanced one pass, 41229 github, level 4 with dict dms, advanced one pass, 41229 github, level 4 with dict dds, advanced one pass, 41229 -github, level 4 with dict copy, advanced one pass, 41216 +github, level 4 with dict copy, advanced one pass, 41205 github, level 4 with dict load, advanced one pass, 41548 github, level 5 row 1, advanced one pass, 134584 github, level 5 row 1 with dict dms, advanced one pass, 38754 @@ -428,11 +428,11 @@ github.tar, level -3 with dict, advanced github.tar, level -1, advanced one pass, 42606 github.tar, level -1 with dict, advanced one pass, 41397 github.tar, level 0, advanced one pass, 38884 -github.tar, level 0 with dict, advanced one pass, 37995 +github.tar, level 0 with dict, advanced one pass, 38380 github.tar, level 0 with dict dms, advanced one pass, 38114 github.tar, level 0 with dict dds, advanced one pass, 38114 -github.tar, level 0 with dict copy, advanced one pass, 37995 -github.tar, level 0 with dict load, advanced one pass, 37956 +github.tar, level 0 with dict copy, advanced one pass, 38380 +github.tar, level 0 with dict load, advanced one pass, 38287 github.tar, level 1, advanced one pass, 39200 github.tar, level 1 with dict, advanced one pass, 38189 github.tar, level 1 with dict dms, advanced one pass, 38398 @@ -440,17 +440,17 @@ github.tar, level 1 with dict dds, advanced github.tar, level 1 with dict copy, advanced one pass, 38189 github.tar, level 1 with dict load, advanced one pass, 38393 github.tar, level 3, advanced one pass, 38884 -github.tar, level 3 with dict, advanced one pass, 37995 +github.tar, level 3 with dict, advanced one pass, 38380 github.tar, level 3 with dict dms, advanced one pass, 38114 github.tar, level 3 with dict dds, advanced one pass, 38114 -github.tar, level 3 with dict copy, advanced one pass, 37995 -github.tar, level 3 with dict load, advanced one pass, 37956 +github.tar, level 3 with dict copy, advanced one pass, 38380 +github.tar, level 3 with dict load, advanced one pass, 38287 github.tar, level 4, advanced one pass, 38880 -github.tar, level 4 with dict, advanced one pass, 37948 +github.tar, level 4 with dict, advanced one pass, 38293 github.tar, level 4 with dict dms, advanced one pass, 37995 github.tar, level 4 with dict dds, advanced one pass, 37995 -github.tar, level 4 with dict copy, advanced one pass, 37948 -github.tar, level 4 with dict load, advanced one pass, 37927 +github.tar, level 4 with dict copy, advanced one pass, 38293 +github.tar, level 4 with dict load, advanced one pass, 38298 github.tar, level 5 row 1, advanced one pass, 39651 github.tar, level 5 row 1 with dict dms, advanced one pass, 39043 github.tar, level 5 row 1 with dict dds, advanced one pass, 39069 @@ -534,7 +534,7 @@ github.tar, level 19 with dict dds, advanced github.tar, level 19 with dict copy, advanced one pass, 32701 github.tar, level 19 with dict load, advanced one pass, 32428 github.tar, no source size, advanced one pass, 38884 -github.tar, no source size with dict, advanced one pass, 37995 +github.tar, no source size with dict, advanced one pass, 38380 github.tar, long distance mode, advanced one pass, 40156 github.tar, multithreaded, advanced one pass, 38884 github.tar, multithreaded long distance mode, advanced one pass, 40139 @@ -624,8 +624,8 @@ github, level 0, advanced github, level 0 with dict, advanced one pass small out, 41118 github, level 0 with dict dms, advanced one pass small out, 41118 github, level 0 with dict dds, advanced one pass small out, 41118 -github, level 0 with dict copy, advanced one pass small out, 41124 -github, level 0 with dict load, advanced one pass small out, 41847 +github, level 0 with dict copy, advanced one pass small out, 41122 +github, level 0 with dict load, advanced one pass small out, 42173 github, level 1, advanced one pass small out, 142365 github, level 1 with dict, advanced one pass small out, 41266 github, level 1 with dict dms, advanced one pass small out, 41266 @@ -636,13 +636,13 @@ github, level 3, advanced github, level 3 with dict, advanced one pass small out, 41118 github, level 3 with dict dms, advanced one pass small out, 41118 github, level 3 with dict dds, advanced one pass small out, 41118 -github, level 3 with dict copy, advanced one pass small out, 41124 -github, level 3 with dict load, advanced one pass small out, 41847 +github, level 3 with dict copy, advanced one pass small out, 41122 +github, level 3 with dict load, advanced one pass small out, 42173 github, level 4, advanced one pass small out, 136199 github, level 4 with dict, advanced one pass small out, 41229 github, level 4 with dict dms, advanced one pass small out, 41229 github, level 4 with dict dds, advanced one pass small out, 41229 -github, level 4 with dict copy, advanced one pass small out, 41216 +github, level 4 with dict copy, advanced one pass small out, 41205 github, level 4 with dict load, advanced one pass small out, 41548 github, level 5 row 1, advanced one pass small out, 134584 github, level 5 row 1 with dict dms, advanced one pass small out, 38754 @@ -746,11 +746,11 @@ github.tar, level -3 with dict, advanced github.tar, level -1, advanced one pass small out, 42606 github.tar, level -1 with dict, advanced one pass small out, 41397 github.tar, level 0, advanced one pass small out, 38884 -github.tar, level 0 with dict, advanced one pass small out, 37995 +github.tar, level 0 with dict, advanced one pass small out, 38380 github.tar, level 0 with dict dms, advanced one pass small out, 38114 github.tar, level 0 with dict dds, advanced one pass small out, 38114 -github.tar, level 0 with dict copy, advanced one pass small out, 37995 -github.tar, level 0 with dict load, advanced one pass small out, 37956 +github.tar, level 0 with dict copy, advanced one pass small out, 38380 +github.tar, level 0 with dict load, advanced one pass small out, 38287 github.tar, level 1, advanced one pass small out, 39200 github.tar, level 1 with dict, advanced one pass small out, 38189 github.tar, level 1 with dict dms, advanced one pass small out, 38398 @@ -758,17 +758,17 @@ github.tar, level 1 with dict dds, advanced github.tar, level 1 with dict copy, advanced one pass small out, 38189 github.tar, level 1 with dict load, advanced one pass small out, 38393 github.tar, level 3, advanced one pass small out, 38884 -github.tar, level 3 with dict, advanced one pass small out, 37995 +github.tar, level 3 with dict, advanced one pass small out, 38380 github.tar, level 3 with dict dms, advanced one pass small out, 38114 github.tar, level 3 with dict dds, advanced one pass small out, 38114 -github.tar, level 3 with dict copy, advanced one pass small out, 37995 -github.tar, level 3 with dict load, advanced one pass small out, 37956 +github.tar, level 3 with dict copy, advanced one pass small out, 38380 +github.tar, level 3 with dict load, advanced one pass small out, 38287 github.tar, level 4, advanced one pass small out, 38880 -github.tar, level 4 with dict, advanced one pass small out, 37948 +github.tar, level 4 with dict, advanced one pass small out, 38293 github.tar, level 4 with dict dms, advanced one pass small out, 37995 github.tar, level 4 with dict dds, advanced one pass small out, 37995 -github.tar, level 4 with dict copy, advanced one pass small out, 37948 -github.tar, level 4 with dict load, advanced one pass small out, 37927 +github.tar, level 4 with dict copy, advanced one pass small out, 38293 +github.tar, level 4 with dict load, advanced one pass small out, 38298 github.tar, level 5 row 1, advanced one pass small out, 39651 github.tar, level 5 row 1 with dict dms, advanced one pass small out, 39043 github.tar, level 5 row 1 with dict dds, advanced one pass small out, 39069 @@ -852,7 +852,7 @@ github.tar, level 19 with dict dds, advanced github.tar, level 19 with dict copy, advanced one pass small out, 32701 github.tar, level 19 with dict load, advanced one pass small out, 32428 github.tar, no source size, advanced one pass small out, 38884 -github.tar, no source size with dict, advanced one pass small out, 37995 +github.tar, no source size with dict, advanced one pass small out, 38380 github.tar, long distance mode, advanced one pass small out, 40156 github.tar, multithreaded, advanced one pass small out, 38884 github.tar, multithreaded long distance mode, advanced one pass small out, 40139 @@ -901,10 +901,10 @@ silesia, multithreaded with advanced params, advanced silesia.tar, level -5, advanced streaming, 6853184 silesia.tar, level -3, advanced streaming, 6503455 silesia.tar, level -1, advanced streaming, 6175761 -silesia.tar, level 0, advanced streaming, 4846783 +silesia.tar, level 0, advanced streaming, 4839587 silesia.tar, level 1, advanced streaming, 5306719 -silesia.tar, level 3, advanced streaming, 4846783 -silesia.tar, level 4, advanced streaming, 4785332 +silesia.tar, level 3, advanced streaming, 4839587 +silesia.tar, level 4, advanced streaming, 4777299 silesia.tar, level 5 row 1, advanced streaming, 4664523 silesia.tar, level 5 row 2, advanced streaming, 4668292 silesia.tar, level 5, advanced streaming, 4664523 @@ -920,7 +920,7 @@ silesia.tar, level 12 row 2, advanced silesia.tar, level 13, advanced streaming, 4486652 silesia.tar, level 16, advanced streaming, 4358029 silesia.tar, level 19, advanced streaming, 4258228 -silesia.tar, no source size, advanced streaming, 4846779 +silesia.tar, no source size, advanced streaming, 4839583 silesia.tar, long distance mode, advanced streaming, 4825842 silesia.tar, multithreaded, advanced streaming, 4836000 silesia.tar, multithreaded long distance mode, advanced streaming, 4827826 @@ -928,7 +928,7 @@ silesia.tar, small window log, advanced silesia.tar, small hash log, advanced streaming, 6529503 silesia.tar, small chain log, advanced streaming, 4915956 silesia.tar, explicit params, advanced streaming, 4791739 -silesia.tar, uncompressed literals, advanced streaming, 5123274 +silesia.tar, uncompressed literals, advanced streaming, 5115568 silesia.tar, uncompressed literals optimal, advanced streaming, 4306968 silesia.tar, huffman literals, advanced streaming, 5323245 silesia.tar, multithreaded with advanced params, advanced streaming, 5116579 @@ -942,8 +942,8 @@ github, level 0, advanced github, level 0 with dict, advanced streaming, 41118 github, level 0 with dict dms, advanced streaming, 41118 github, level 0 with dict dds, advanced streaming, 41118 -github, level 0 with dict copy, advanced streaming, 41124 -github, level 0 with dict load, advanced streaming, 41847 +github, level 0 with dict copy, advanced streaming, 41122 +github, level 0 with dict load, advanced streaming, 42173 github, level 1, advanced streaming, 142365 github, level 1 with dict, advanced streaming, 41266 github, level 1 with dict dms, advanced streaming, 41266 @@ -954,13 +954,13 @@ github, level 3, advanced github, level 3 with dict, advanced streaming, 41118 github, level 3 with dict dms, advanced streaming, 41118 github, level 3 with dict dds, advanced streaming, 41118 -github, level 3 with dict copy, advanced streaming, 41124 -github, level 3 with dict load, advanced streaming, 41847 +github, level 3 with dict copy, advanced streaming, 41122 +github, level 3 with dict load, advanced streaming, 42173 github, level 4, advanced streaming, 136199 github, level 4 with dict, advanced streaming, 41229 github, level 4 with dict dms, advanced streaming, 41229 github, level 4 with dict dds, advanced streaming, 41229 -github, level 4 with dict copy, advanced streaming, 41216 +github, level 4 with dict copy, advanced streaming, 41205 github, level 4 with dict load, advanced streaming, 41548 github, level 5 row 1, advanced streaming, 134584 github, level 5 row 1 with dict dms, advanced streaming, 38754 @@ -1064,11 +1064,11 @@ github.tar, level -3 with dict, advanced github.tar, level -1, advanced streaming, 42687 github.tar, level -1 with dict, advanced streaming, 41486 github.tar, level 0, advanced streaming, 38884 -github.tar, level 0 with dict, advanced streaming, 37995 +github.tar, level 0 with dict, advanced streaming, 38380 github.tar, level 0 with dict dms, advanced streaming, 38114 github.tar, level 0 with dict dds, advanced streaming, 38114 -github.tar, level 0 with dict copy, advanced streaming, 37995 -github.tar, level 0 with dict load, advanced streaming, 37956 +github.tar, level 0 with dict copy, advanced streaming, 38380 +github.tar, level 0 with dict load, advanced streaming, 38287 github.tar, level 1, advanced streaming, 39346 github.tar, level 1 with dict, advanced streaming, 38251 github.tar, level 1 with dict dms, advanced streaming, 38557 @@ -1076,17 +1076,17 @@ github.tar, level 1 with dict dds, advanced github.tar, level 1 with dict copy, advanced streaming, 38251 github.tar, level 1 with dict load, advanced streaming, 38503 github.tar, level 3, advanced streaming, 38884 -github.tar, level 3 with dict, advanced streaming, 37995 +github.tar, level 3 with dict, advanced streaming, 38380 github.tar, level 3 with dict dms, advanced streaming, 38114 github.tar, level 3 with dict dds, advanced streaming, 38114 -github.tar, level 3 with dict copy, advanced streaming, 37995 -github.tar, level 3 with dict load, advanced streaming, 37956 +github.tar, level 3 with dict copy, advanced streaming, 38380 +github.tar, level 3 with dict load, advanced streaming, 38287 github.tar, level 4, advanced streaming, 38880 -github.tar, level 4 with dict, advanced streaming, 37948 +github.tar, level 4 with dict, advanced streaming, 38293 github.tar, level 4 with dict dms, advanced streaming, 37995 github.tar, level 4 with dict dds, advanced streaming, 37995 -github.tar, level 4 with dict copy, advanced streaming, 37948 -github.tar, level 4 with dict load, advanced streaming, 37927 +github.tar, level 4 with dict copy, advanced streaming, 38293 +github.tar, level 4 with dict load, advanced streaming, 38298 github.tar, level 5 row 1, advanced streaming, 39651 github.tar, level 5 row 1 with dict dms, advanced streaming, 39043 github.tar, level 5 row 1 with dict dds, advanced streaming, 39069 @@ -1203,10 +1203,10 @@ silesia, huffman literals, old stre silesia.tar, level -5, old streaming, 6853184 silesia.tar, level -3, old streaming, 6503455 silesia.tar, level -1, old streaming, 6175761 -silesia.tar, level 0, old streaming, 4846783 +silesia.tar, level 0, old streaming, 4839587 silesia.tar, level 1, old streaming, 5306719 -silesia.tar, level 3, old streaming, 4846783 -silesia.tar, level 4, old streaming, 4785332 +silesia.tar, level 3, old streaming, 4839587 +silesia.tar, level 4, old streaming, 4777299 silesia.tar, level 5, old streaming, 4664523 silesia.tar, level 6, old streaming, 4599420 silesia.tar, level 7, old streaming, 4565332 @@ -1214,8 +1214,8 @@ silesia.tar, level 9, old stre silesia.tar, level 13, old streaming, 4486652 silesia.tar, level 16, old streaming, 4358029 silesia.tar, level 19, old streaming, 4258228 -silesia.tar, no source size, old streaming, 4846779 -silesia.tar, uncompressed literals, old streaming, 4846783 +silesia.tar, no source size, old streaming, 4839583 +silesia.tar, uncompressed literals, old streaming, 4839587 silesia.tar, uncompressed literals optimal, old streaming, 4258228 silesia.tar, huffman literals, old streaming, 6175761 github, level -5, old streaming, 204407 @@ -1258,13 +1258,13 @@ github.tar, level -3 with dict, old stre github.tar, level -1, old streaming, 42687 github.tar, level -1 with dict, old streaming, 41486 github.tar, level 0, old streaming, 38884 -github.tar, level 0 with dict, old streaming, 37995 +github.tar, level 0 with dict, old streaming, 38380 github.tar, level 1, old streaming, 39346 github.tar, level 1 with dict, old streaming, 38251 github.tar, level 3, old streaming, 38884 -github.tar, level 3 with dict, old streaming, 37995 +github.tar, level 3 with dict, old streaming, 38380 github.tar, level 4, old streaming, 38880 -github.tar, level 4 with dict, old streaming, 37948 +github.tar, level 4 with dict, old streaming, 38293 github.tar, level 5, old streaming, 39651 github.tar, level 5 with dict, old streaming, 39145 github.tar, level 6, old streaming, 39282 @@ -1313,10 +1313,10 @@ silesia, multithreaded with advanced params, old stre silesia.tar, level -5, old streaming advanced, 6853184 silesia.tar, level -3, old streaming advanced, 6503455 silesia.tar, level -1, old streaming advanced, 6175761 -silesia.tar, level 0, old streaming advanced, 4846783 +silesia.tar, level 0, old streaming advanced, 4839587 silesia.tar, level 1, old streaming advanced, 5306719 -silesia.tar, level 3, old streaming advanced, 4846783 -silesia.tar, level 4, old streaming advanced, 4785332 +silesia.tar, level 3, old streaming advanced, 4839587 +silesia.tar, level 4, old streaming advanced, 4777299 silesia.tar, level 5, old streaming advanced, 4664523 silesia.tar, level 6, old streaming advanced, 4599420 silesia.tar, level 7, old streaming advanced, 4565332 @@ -1324,18 +1324,18 @@ silesia.tar, level 9, old stre silesia.tar, level 13, old streaming advanced, 4486652 silesia.tar, level 16, old streaming advanced, 4358029 silesia.tar, level 19, old streaming advanced, 4258228 -silesia.tar, no source size, old streaming advanced, 4846779 -silesia.tar, long distance mode, old streaming advanced, 4846783 -silesia.tar, multithreaded, old streaming advanced, 4846783 -silesia.tar, multithreaded long distance mode, old streaming advanced, 4846783 +silesia.tar, no source size, old streaming advanced, 4839583 +silesia.tar, long distance mode, old streaming advanced, 4839587 +silesia.tar, multithreaded, old streaming advanced, 4839587 +silesia.tar, multithreaded long distance mode, old streaming advanced, 4839587 silesia.tar, small window log, old streaming advanced, 7117027 silesia.tar, small hash log, old streaming advanced, 6529503 silesia.tar, small chain log, old streaming advanced, 4915956 silesia.tar, explicit params, old streaming advanced, 4791739 -silesia.tar, uncompressed literals, old streaming advanced, 4846783 +silesia.tar, uncompressed literals, old streaming advanced, 4839587 silesia.tar, uncompressed literals optimal, old streaming advanced, 4258228 silesia.tar, huffman literals, old streaming advanced, 6175761 -silesia.tar, multithreaded with advanced params, old streaming advanced, 4846783 +silesia.tar, multithreaded with advanced params, old streaming advanced, 4839587 github, level -5, old streaming advanced, 213265 github, level -5 with dict, old streaming advanced, 46708 github, level -3, old streaming advanced, 196126 @@ -1384,13 +1384,13 @@ github.tar, level -3 with dict, old stre github.tar, level -1, old streaming advanced, 42687 github.tar, level -1 with dict, old streaming advanced, 41762 github.tar, level 0, old streaming advanced, 38884 -github.tar, level 0 with dict, old streaming advanced, 38013 +github.tar, level 0 with dict, old streaming advanced, 38442 github.tar, level 1, old streaming advanced, 39346 github.tar, level 1 with dict, old streaming advanced, 38507 github.tar, level 3, old streaming advanced, 38884 -github.tar, level 3 with dict, old streaming advanced, 38013 +github.tar, level 3 with dict, old streaming advanced, 38442 github.tar, level 4, old streaming advanced, 38880 -github.tar, level 4 with dict, old streaming advanced, 38063 +github.tar, level 4 with dict, old streaming advanced, 38473 github.tar, level 5, old streaming advanced, 39651 github.tar, level 5 with dict, old streaming advanced, 39018 github.tar, level 6, old streaming advanced, 39282 @@ -1436,10 +1436,10 @@ github, no source size with dict, old stre github.tar, level -5 with dict, old streaming cdict, 51407 github.tar, level -3 with dict, old streaming cdict, 45254 github.tar, level -1 with dict, old streaming cdict, 41973 -github.tar, level 0 with dict, old streaming cdict, 37956 +github.tar, level 0 with dict, old streaming cdict, 38287 github.tar, level 1 with dict, old streaming cdict, 38503 -github.tar, level 3 with dict, old streaming cdict, 37956 -github.tar, level 4 with dict, old streaming cdict, 37927 +github.tar, level 3 with dict, old streaming cdict, 38287 +github.tar, level 4 with dict, old streaming cdict, 38298 github.tar, level 5 with dict, old streaming cdict, 39000 github.tar, level 6 with dict, old streaming cdict, 38647 github.tar, level 7 with dict, old streaming cdict, 37286 @@ -1466,10 +1466,10 @@ github, no source size with dict, old stre github.tar, level -5 with dict, old streaming advanced cdict, 50907 github.tar, level -3 with dict, old streaming advanced cdict, 45032 github.tar, level -1 with dict, old streaming advanced cdict, 41589 -github.tar, level 0 with dict, old streaming advanced cdict, 38013 +github.tar, level 0 with dict, old streaming advanced cdict, 38442 github.tar, level 1 with dict, old streaming advanced cdict, 38294 -github.tar, level 3 with dict, old streaming advanced cdict, 38013 -github.tar, level 4 with dict, old streaming advanced cdict, 38063 +github.tar, level 3 with dict, old streaming advanced cdict, 38442 +github.tar, level 4 with dict, old streaming advanced cdict, 38473 github.tar, level 5 with dict, old streaming advanced cdict, 39018 github.tar, level 6 with dict, old streaming advanced cdict, 38635 github.tar, level 7 with dict, old streaming advanced cdict, 37264