diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index e233bc5ff1b..3fe9ad120bf 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -660,11 +660,95 @@ ZSTD_selectAddr(U32 index, U32 lowLimit, const BYTE* candidate, const BYTE* back : "r"(index), "r"(lowLimit), "r"(backup) ); return candidate; +#elif defined(__GNUC__) && defined(__aarch64__) + __asm__ ( + "cmp %w1, %w2\n" + "csel %0, %3, %0, lo\n" + : "+r"(candidate) + : "r"(index), "r"(lowLimit), "r"(backup) + : "cc"); + return candidate; #else return index >= lowLimit ? candidate : backup; #endif } +/* ZSTD_match4Found_* / ZSTD_match8Found_* : + * @return 1 when the 4 (resp. 8) bytes at matchAddress equal those at currentPtr, + * for a hash-table candidate that may only be dereferenced when matchIdx >= idxLowLimit. + * The _cmov variants never branch on index validity: an out-of-range candidate is + * redirected to a dummy buffer, so the only branch left is on the byte comparison, + * which is rarely taken and predicts well. They are the right choice when validity + * is unpredictable, i.e. when the window is not much larger than the tables, so + * that many entries are stale, and in extDict mode where the valid range moves + * with every block. + * The _branch variants test validity first; they are faster when nearly every + * candidate is within range (large windows), where that branch is free and the + * select would only lengthen the dependency chain in front of the candidate load. */ +typedef int (*ZSTD_matchFound)(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit); + +MEM_STATIC int +ZSTD_match4Found_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) +{ + /* Array of ~random data, should have low probability of matching data. + * Load from here if the index is invalid. + * Used to avoid unpredictable branches. */ + static const BYTE dummy[] = {0x12,0x34,0x56,0x78}; + + const BYTE* mvalAddr = ZSTD_selectAddr(matchIdx, idxLowLimit, matchAddress, dummy); + /* Note: this used to be written as : return test1 && test2; + * Unfortunately, once inlined, these tests become branches, + * in which case it becomes critical that they are executed in the right order (test1 then test2). + * So we have to write these tests in a specific manner to ensure their ordering. + */ + if (MEM_read32(currentPtr) != MEM_read32(mvalAddr)) return 0; + /* force ordering of these tests, which matters once the function is inlined, as they become branches */ +#if defined(__GNUC__) + __asm__(""); +#endif + return matchIdx >= idxLowLimit; +} + +MEM_STATIC int +ZSTD_match4Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) +{ + /* using a branch instead of a cmov, + * because it's faster in scenarios where matchIdx >= idxLowLimit is generally true, + * aka almost all candidates are within range */ + U32 mval; + if (matchIdx >= idxLowLimit) { + mval = MEM_read32(matchAddress); + } else { + mval = MEM_read32(currentPtr) ^ 1; /* guaranteed to not match. */ + } + + return (MEM_read32(currentPtr) == mval); +} + +MEM_STATIC int +ZSTD_match8Found_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) +{ + static const BYTE dummy[] = {0x12,0x34,0x56,0x78,0x9a,0xbc,0xde,0xf0}; + const BYTE* mvalAddr = ZSTD_selectAddr(matchIdx, idxLowLimit, matchAddress, dummy); + if (MEM_read64(currentPtr) != MEM_read64(mvalAddr)) return 0; +#if defined(__GNUC__) + __asm__(""); +#endif + return matchIdx >= idxLowLimit; +} + +MEM_STATIC int +ZSTD_match8Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) +{ + U64 mval; + if (matchIdx >= idxLowLimit) { + mval = MEM_read64(matchAddress); + } else { + mval = MEM_read64(currentPtr) ^ 1; /* guaranteed to not match. */ + } + return (MEM_read64(currentPtr) == mval); +} + /* ZSTD_noCompressBlock() : * Writes uncompressed block to dst buffer from given src. * Returns the size of the block */ @@ -913,6 +997,45 @@ ZSTD_count_2segments(const BYTE* ip, const BYTE* match, return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd); } +/* ZSTD_count_inline(), ZSTD_count_2segments_inline() : + * same results as ZSTD_count() / ZSTD_count_2segments(), with inlining forced. + * For block compressors whose translation unit carries enough template + * specializations that the compiler's unit-growth limits would otherwise turn + * these few hot instructions into calls (seen with GCC 13-15 at -O3 on + * zstd_double_fast.c). Kept separate so that code generation for every other + * caller of ZSTD_count() is unchanged. */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_count_inline(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) +{ + const BYTE* const pStart = pIn; + const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1); + + if (pIn < pInLoopLimit) { + { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (diff) return ZSTD_NbCommonBytes(diff); } + pIn+=sizeof(size_t); pMatch+=sizeof(size_t); + while (pIn < pInLoopLimit) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } + pIn += ZSTD_NbCommonBytes(diff); + return (size_t)(pIn - pStart); + } } + if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIncParams; U32* const hashLong = ms->hashTable; @@ -142,14 +144,12 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( const BYTE* matchl0; /* the long match for ip */ const BYTE* matchs0; /* the short match for ip */ const BYTE* matchl1; /* the long match for ip1 */ - const BYTE* matchs0_safe; /* matchs0 or safe address */ const BYTE* ip = istart; /* the current position */ const BYTE* ip1; /* the next position */ - /* Array of ~random data, should have low probability of matching data - * we load from here instead of from tables, if matchl0/matchl1 are - * invalid indices. Used to avoid unpredictable branches. */ - const BYTE dummy[] = {0x12,0x34,0x56,0x78,0x9a,0xbc,0xde,0xf0,0xe2,0xb4}; + + const ZSTD_matchFound matchLongFound = useCmov ? ZSTD_match8Found_cmov : ZSTD_match8Found_branch; + const ZSTD_matchFound matchShortFound = useCmov ? ZSTD_match4Found_cmov : ZSTD_match4Found_branch; DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic"); @@ -188,7 +188,7 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( /* check noDict repcode */ if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { - mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + mLength = ZSTD_count_inline(ip+1+4, ip+1+4-offset_1, iend) + 4; ip++; ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); goto _match_stored; @@ -196,29 +196,20 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( hl1 = ZSTD_hashPtr(ip1, hBitsL, 8); - /* idxl0 > prefixLowestIndex is a (somewhat) unpredictable branch. - * However expression below complies into conditional move. Since - * match is unlikely and we only *branch* on idxl0 > prefixLowestIndex - * if there is a match, all branches become predictable. */ - { const BYTE* const matchl0_safe = ZSTD_selectAddr(idxl0, prefixLowestIndex, matchl0, &dummy[0]); - - /* check prefix long match */ - if (MEM_read64(matchl0_safe) == MEM_read64(ip) && matchl0_safe == matchl0) { - mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8; - offset = (U32)(ip-matchl0); - while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ - goto _match_found; - } } + /* check prefix long match */ + if (matchLongFound(ip, matchl0, idxl0, prefixLowestIndex)) { + mLength = ZSTD_count_inline(ip+8, matchl0+8, iend) + 8; + offset = (U32)(ip-matchl0); + while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ + goto _match_found; + } idxl1 = hashLong[hl1]; matchl1 = base + idxl1; - /* Same optimization as matchl0 above */ - matchs0_safe = ZSTD_selectAddr(idxs0, prefixLowestIndex, matchs0, &dummy[0]); - /* check prefix short match */ - if(MEM_read32(matchs0_safe) == MEM_read32(ip) && matchs0_safe == matchs0) { - goto _search_next_long; + if (matchShortFound(ip, matchs0, idxs0, prefixLowestIndex)) { + goto _search_next_long; } if (ip1 >= nextStep) { @@ -253,12 +244,12 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( _search_next_long: /* short match found: let's check for a longer one */ - mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4; + mLength = ZSTD_count_inline(ip+4, matchs0+4, iend) + 4; offset = (U32)(ip - matchs0); /* check long match at +1 position */ - if ((idxl1 > prefixLowestIndex) && (MEM_read64(matchl1) == MEM_read64(ip1))) { - size_t const l1len = ZSTD_count(ip1+8, matchl1+8, iend) + 8; + if (matchLongFound(ip1, matchl1, idxl1, prefixLowestIndex)) { + size_t const l1len = ZSTD_count_inline(ip1+8, matchl1+8, iend) + 8; if (l1len > mLength) { /* use the long match instead */ ip = ip1; @@ -309,7 +300,7 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( && ( (offset_2>0) & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { /* store sequence */ - size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + size_t const rLength = ZSTD_count_inline(ip+4, ip+4-offset_2, iend) + 4; U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); @@ -328,7 +319,7 @@ ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize, - U32 const mls /* template */) + U32 const mls /* template */, int const useCmov) { ZSTD_compressionParameters const* cParams = &ms->cParams; U32* const hashLong = ms->hashTable; @@ -361,6 +352,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic"); + (void)useCmov; /* if a dictionary is attached, it must be within window range */ assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); @@ -407,7 +399,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex)) && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + mLength = ZSTD_count_2segments_inline(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; ip++; ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); goto _match_stored; @@ -415,7 +407,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( if ((matchIndexL >= prefixLowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { /* check prefix long match */ - mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; + mLength = ZSTD_count_inline(ip+8, matchLong+8, iend) + 8; offset = (U32)(ip-matchLong); while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ goto _match_found; @@ -426,7 +418,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( assert(dictMatchL < dictEnd); if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) { - mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8; + mLength = ZSTD_count_2segments_inline(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8; offset = (U32)(curr - dictMatchIndexL - dictIndexDelta); while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ goto _match_found; @@ -464,7 +456,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( /* check prefix long +1 match */ if ((matchIndexL3 >= prefixLowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1))) { - mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; + mLength = ZSTD_count_inline(ip+9, matchL3+8, iend) + 8; ip++; offset = (U32)(ip-matchL3); while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ @@ -475,7 +467,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3; assert(dictMatchL3 < dictEnd); if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) { - mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8; + mLength = ZSTD_count_2segments_inline(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8; ip++; offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta); while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ @@ -484,11 +476,11 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( /* if no long +1 match, explore the short match we found */ if (matchIndexS < prefixLowestIndex) { - mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; + mLength = ZSTD_count_2segments_inline(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; offset = (U32)(curr - matchIndexS); while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ } else { - mLength = ZSTD_count(ip+4, match+4, iend) + 4; + mLength = ZSTD_count_inline(ip+4, match+4, iend) + 4; offset = (U32)(ip - match); while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ } @@ -524,7 +516,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex2)) && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; + size_t const repLength2 = ZSTD_count_2segments_inline(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; @@ -546,23 +538,28 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( return (size_t)(iend - anchor); } -#define ZSTD_GEN_DFAST_FN(dictMode, mls) \ - static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ - void const* src, size_t srcSize) \ - { \ - return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \ +#define ZSTD_GEN_DFAST_FN(dictMode, mls, cmov) \ + static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls##_##cmov( \ + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls, cmov); \ } -ZSTD_GEN_DFAST_FN(noDict, 4) -ZSTD_GEN_DFAST_FN(noDict, 5) -ZSTD_GEN_DFAST_FN(noDict, 6) -ZSTD_GEN_DFAST_FN(noDict, 7) +ZSTD_GEN_DFAST_FN(noDict, 4, 1) +ZSTD_GEN_DFAST_FN(noDict, 5, 1) +ZSTD_GEN_DFAST_FN(noDict, 6, 1) +ZSTD_GEN_DFAST_FN(noDict, 7, 1) + +ZSTD_GEN_DFAST_FN(noDict, 4, 0) +ZSTD_GEN_DFAST_FN(noDict, 5, 0) +ZSTD_GEN_DFAST_FN(noDict, 6, 0) +ZSTD_GEN_DFAST_FN(noDict, 7, 0) -ZSTD_GEN_DFAST_FN(dictMatchState, 4) -ZSTD_GEN_DFAST_FN(dictMatchState, 5) -ZSTD_GEN_DFAST_FN(dictMatchState, 6) -ZSTD_GEN_DFAST_FN(dictMatchState, 7) +ZSTD_GEN_DFAST_FN(dictMatchState, 4, 0) +ZSTD_GEN_DFAST_FN(dictMatchState, 5, 0) +ZSTD_GEN_DFAST_FN(dictMatchState, 6, 0) +ZSTD_GEN_DFAST_FN(dictMatchState, 7, 0) size_t ZSTD_compressBlock_doubleFast( @@ -570,17 +567,40 @@ size_t ZSTD_compressBlock_doubleFast( void const* src, size_t srcSize) { const U32 mls = ms->cParams.minMatch; - switch(mls) - { - default: /* includes case 3 */ - case 4 : - return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); - case 5 : - return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); - case 6 : - return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); - case 7 : - return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); + /* A table entry goes stale (falls out of the window) when its slot is not + * overwritten for a whole window span. With about one insertion per position + * that is rare once the window is much larger than the tables, and the + * "candidate in range" test then predicts perfectly: use a branch. + * Otherwise (small windows, or tables sized close to the window) many + * entries are stale and the test is unpredictable: use a cmov. */ + int const useCmov = ms->cParams.windowLog < ms->cParams.hashLog + 4; + if (useCmov) { + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_noDict_4_1(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_noDict_5_1(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_noDict_6_1(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_noDict_7_1(ms, seqStore, rep, src, srcSize); + } + } else { + /* use a branch instead */ + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_noDict_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_noDict_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_noDict_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_noDict_7_0(ms, seqStore, rep, src, srcSize); + } } } @@ -594,18 +614,18 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); + return ZSTD_compressBlock_doubleFast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); + return ZSTD_compressBlock_doubleFast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); + return ZSTD_compressBlock_doubleFast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); + return ZSTD_compressBlock_doubleFast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize); } } -static +FORCE_INLINE_TEMPLATE ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_compressBlock_doubleFast_extDict_generic( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], @@ -618,7 +638,6 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( U32* const hashSmall = ms->chainTable; U32 const hBitsS = cParams->chainLog; const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; const BYTE* anchor = istart; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; @@ -633,6 +652,39 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* const dictStart = dictBase + dictStartIndex; const BYTE* const dictEnd = dictBase + prefixStartIndex; U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + size_t mLength; + U32 offset; + U32 curr; + + /* how many positions to search before increasing step size */ + const size_t kStepIncr = 1 << kSearchStrength; + /* the position at which to increment the step size if no match is found */ + const BYTE* nextStep; + size_t step; /* the current step size */ + + size_t hl0; /* the long hash at ip */ + size_t hl1; /* the long hash at ip1 */ + + U32 idxl0; /* the long match index for ip */ + U32 idxs0; /* the short match index for ip */ + U32 idxl1; /* the long match index for ip1 */ + + const BYTE* matchl0; /* the long match for ip */ + const BYTE* matchs0; /* the short match for ip */ + const BYTE* matchl1; /* the long match for ip1 */ + const BYTE* matchEnd; /* end of the segment containing the match */ + const BYTE* matchLow; /* start of the segment containing the match */ + + const BYTE* ip = istart; /* the current position */ + const BYTE* ip1; /* the next position */ + + /* In extDict mode the lowest valid index moves with every block, so + * the tables always hold a mix of in-range and stale entries: test + * candidates without branching on index validity, whatever the window size. */ + const ZSTD_matchFound matchLongFound = ZSTD_match8Found_cmov; + const ZSTD_matchFound matchShortFound = ZSTD_match4Found_cmov; DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize); @@ -640,75 +692,157 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( if (prefixStartIndex == dictStartIndex) return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ - const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); - const U32 matchIndex = hashSmall[hSmall]; - const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; - const BYTE* match = matchBase + matchIndex; - - const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); - const U32 matchLongIndex = hashLong[hLong]; - const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base; - const BYTE* matchLong = matchLongBase + matchLongIndex; - - const U32 curr = (U32)(ip-base); - const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ - const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - size_t mLength; - hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ + /* A table index is dereferenceable iff it is >= dictStartIndex; indexes + * below prefixStartIndex are addressed through dictBase, the others + * through base. Every position was inserted with at least 8 bytes left + * in its own segment, so an in-range candidate can always be read with + * MEM_read64() without running past dictEnd. */ - if (((ZSTD_index_overlap_check(prefixStartIndex, repIndex)) - & (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */ - && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); - } else { - if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { - const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; - const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart; - U32 offset; - mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8; - offset = curr - matchLongIndex; - while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); - - } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { - size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); - U32 const matchIndex3 = hashLong[h3]; - const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base; - const BYTE* match3 = match3Base + matchIndex3; - U32 offset; - hashLong[h3] = curr + 1; - if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { - const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend; - const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart; - mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8; - ip++; - offset = curr+1 - matchIndex3; - while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + { U32 const current = (U32)(ip - base); + U32 const maxRep = current - dictStartIndex; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } + + /* Outer Loop: one iteration per match found and stored */ + while (1) { + step = 1; + nextStep = ip + kStepIncr; + ip1 = ip + step; + + if (ip1 > ilimit) { + goto _cleanup; + } + + hl0 = ZSTD_hashPtr(ip, hBitsL, 8); + idxl0 = hashLong[hl0]; + matchl0 = (idxl0 < prefixStartIndex ? dictBase : base) + idxl0; + + /* Inner Loop: one iteration per search / position */ + do { + const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 repIndex = (U32)(ip-base) + 1 - offset_1; + const BYTE* const repMatch = (repIndex < prefixStartIndex ? dictBase : base) + repIndex; + idxs0 = hashSmall[hs0]; + curr = (U32)(ip-base); + matchs0 = (idxs0 < prefixStartIndex ? dictBase : base) + idxs0; + + hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */ + + /* check repcode at ip+1 */ + if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex)) & (offset_1 > 0)) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments_inline(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + goto _match_stored; + } + + hl1 = ZSTD_hashPtr(ip1, hBitsL, 8); + + /* check long match */ + if (matchLongFound(ip, matchl0, idxl0, dictStartIndex)) { + if (idxl0 < prefixStartIndex) { + matchEnd = dictEnd; + matchLow = dictStart; } else { - const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; - const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; - mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; - offset = curr - matchIndex; - while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + matchEnd = iend; + matchLow = prefixStart; } - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + mLength = ZSTD_count_2segments_inline(ip+8, matchl0+8, iend, matchEnd, prefixStart) + 8; + offset = curr - idxl0; + while (((ip>anchor) & (matchl0>matchLow)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ + goto _match_found; + } - } else { - ip += ((ip-anchor) >> kSearchStrength) + 1; - continue; - } } + idxl1 = hashLong[hl1]; + matchl1 = (idxl1 < prefixStartIndex ? dictBase : base) + idxl1; + + /* check short match */ + if (matchShortFound(ip, matchs0, idxs0, dictStartIndex)) { + if (idxs0 < prefixStartIndex) { + matchEnd = dictEnd; + matchLow = dictStart; + } else { + matchEnd = iend; + matchLow = prefixStart; + } + goto _search_next_long; + } + + if (ip1 >= nextStep) { + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + step++; + nextStep += kStepIncr; + } + ip = ip1; + ip1 += step; - /* move to next sequence start */ + hl0 = hl1; + idxl0 = idxl1; + matchl0 = matchl1; + #if defined(__aarch64__) + PREFETCH_L1(ip+256); + #endif + } while (ip1 <= ilimit); + +_cleanup: + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_search_next_long: + + /* short match found: let's check for a longer one */ + mLength = ZSTD_count_2segments_inline(ip+4, matchs0+4, iend, matchEnd, prefixStart) + 4; + offset = curr - idxs0; + + /* check long match at +1 position */ + if (matchLongFound(ip1, matchl1, idxl1, dictStartIndex)) { + const BYTE* const matchl1End = idxl1 < prefixStartIndex ? dictEnd : iend; + size_t const l1len = ZSTD_count_2segments_inline(ip1+8, matchl1+8, iend, matchl1End, prefixStart) + 8; + if (l1len > mLength) { + /* use the long match instead */ + ip = ip1; + mLength = l1len; + offset = (U32)(ip-base) - idxl1; + matchs0 = matchl1; + matchLow = idxl1 < prefixStartIndex ? dictStart : prefixStart; + } + } + + while (((ip>anchor) & (matchs0>matchLow)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* complete backward */ + + /* fall-through */ + +_match_found: /* requires ip, offset, mLength */ + offset_2 = offset_1; + offset_1 = offset; + + if (step < 4) { + /* It is unsafe to write this value back to the hashtable when ip1 is + * greater than or equal to the new ip we will have after we're done + * processing this match. Rather than perform that test directly + * (ip1 >= ip + mLength), which costs speed in practice, we do a simpler + * more predictable test. The minmatch even if we take a short match is + * 4 bytes, so as long as step, the distance between ip and ip1 + * (initially) is less than 4, we know ip1 < new ip. */ + hashLong[hl1] = (U32)(ip1 - base); + } + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + +_match_stored: + /* match found */ ip += mLength; anchor = ip; @@ -726,12 +860,11 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( while (ip <= ilimit) { U32 const current2 = (U32)(ip-base); U32 const repIndex2 = current2 - offset_2; - const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; - if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) - & (offset_2 <= current2 - dictStartIndex)) + const BYTE* const repMatch2 = (repIndex2 < prefixStartIndex ? dictBase : base) + repIndex2; + if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) & (offset_2 > 0)) && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + size_t const repLength2 = ZSTD_count_2segments_inline(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; @@ -741,20 +874,23 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic( continue; } break; - } } } - - /* save reps for next block */ - rep[0] = offset_1; - rep[1] = offset_2; - - /* Return the last literals size */ - return (size_t)(iend - anchor); + } + } + } } -ZSTD_GEN_DFAST_FN(extDict, 4) -ZSTD_GEN_DFAST_FN(extDict, 5) -ZSTD_GEN_DFAST_FN(extDict, 6) -ZSTD_GEN_DFAST_FN(extDict, 7) +#define ZSTD_GEN_DFAST_EXTDICT_FN(mls) \ + static size_t ZSTD_compressBlock_doubleFast_extDict_##mls( \ + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ + } + +ZSTD_GEN_DFAST_EXTDICT_FN(4) +ZSTD_GEN_DFAST_EXTDICT_FN(5) +ZSTD_GEN_DFAST_EXTDICT_FN(6) +ZSTD_GEN_DFAST_EXTDICT_FN(7) size_t ZSTD_compressBlock_doubleFast_extDict( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c index ee25bcbac8d..178575aa222 100644 --- a/lib/compress/zstd_fast.c +++ b/lib/compress/zstd_fast.c @@ -97,49 +97,6 @@ void ZSTD_fillHashTable(ZSTD_MatchState_t* ms, } -typedef int (*ZSTD_match4Found) (const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit); - -static int -ZSTD_match4Found_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) -{ - /* Array of ~random data, should have low probability of matching data. - * Load from here if the index is invalid. - * Used to avoid unpredictable branches. */ - static const BYTE dummy[] = {0x12,0x34,0x56,0x78}; - - /* currentIdx >= lowLimit is a (somewhat) unpredictable branch. - * However expression below compiles into conditional move. - */ - const BYTE* mvalAddr = ZSTD_selectAddr(matchIdx, idxLowLimit, matchAddress, dummy); - /* Note: this used to be written as : return test1 && test2; - * Unfortunately, once inlined, these tests become branches, - * in which case it becomes critical that they are executed in the right order (test1 then test2). - * So we have to write these tests in a specific manner to ensure their ordering. - */ - if (MEM_read32(currentPtr) != MEM_read32(mvalAddr)) return 0; - /* force ordering of these tests, which matters once the function is inlined, as they become branches */ -#if defined(__GNUC__) - __asm__(""); -#endif - return matchIdx >= idxLowLimit; -} - -static int -ZSTD_match4Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) -{ - /* using a branch instead of a cmov, - * because it's faster in scenarios where matchIdx >= idxLowLimit is generally true, - * aka almost all candidates are within range */ - U32 mval; - if (matchIdx >= idxLowLimit) { - mval = MEM_read32(matchAddress); - } else { - mval = MEM_read32(currentPtr) ^ 1; /* guaranteed to not match. */ - } - - return (MEM_read32(currentPtr) == mval); -} - /** * If you squint hard enough (and ignore repcodes), the search operation at any @@ -232,7 +189,7 @@ size_t ZSTD_compressBlock_fast_noDict_generic( size_t step; const BYTE* nextStep; const size_t kStepIncr = (1 << (kSearchStrength - 1)); - const ZSTD_match4Found matchFound = useCmov ? ZSTD_match4Found_cmov : ZSTD_match4Found_branch; + const ZSTD_matchFound matchFound = useCmov ? ZSTD_match4Found_cmov : ZSTD_match4Found_branch; DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); ip0 += (ip0 == prefixStart); diff --git a/tests/regression/results.csv b/tests/regression/results.csv index c0d4f4ae7ff..5648d75a70a 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -65,13 +65,13 @@ github, level -3 with dict, compress github, level -1, compress cctx, 175468 github, level -1 with dict, compress cctx, 42044 github, level 0, compress cctx, 136331 -github, level 0 with dict, compress cctx, 41534 +github, level 0 with dict, compress cctx, 41475 github, level 1, compress cctx, 142365 github, level 1 with dict, compress cctx, 41715 github, level 3, compress cctx, 136331 -github, level 3 with dict, compress cctx, 41534 +github, level 3 with dict, compress cctx, 41475 github, level 4, compress cctx, 136199 -github, level 4 with dict, compress cctx, 41725 +github, level 4 with dict, compress cctx, 41673 github, level 5, compress cctx, 135121 github, level 5 with dict, compress cctx, 38755 github, level 6, compress cctx, 135122 @@ -194,13 +194,13 @@ github.tar, level -3 with dict, zstdcli, github.tar, level -1, zstdcli, 42680 github.tar, level -1 with dict, zstdcli, 41486 github.tar, level 0, zstdcli, 38888 -github.tar, level 0 with dict, zstdcli, 37999 +github.tar, level 0 with dict, zstdcli, 38384 github.tar, level 1, zstdcli, 39340 github.tar, level 1 with dict, zstdcli, 38230 github.tar, level 3, zstdcli, 38888 -github.tar, level 3 with dict, zstdcli, 37999 +github.tar, level 3 with dict, zstdcli, 38384 github.tar, level 4, zstdcli, 38884 -github.tar, level 4 with dict, zstdcli, 37952 +github.tar, level 4 with dict, zstdcli, 38297 github.tar, level 5, zstdcli, 39655 github.tar, level 5 with dict, zstdcli, 39073 github.tar, level 6, zstdcli, 39286 @@ -306,8 +306,8 @@ github, level 0, advanced github, level 0 with dict, advanced one pass, 41118 github, level 0 with dict dms, advanced one pass, 41118 github, level 0 with dict dds, advanced one pass, 41118 -github, level 0 with dict copy, advanced one pass, 41124 -github, level 0 with dict load, advanced one pass, 41847 +github, level 0 with dict copy, advanced one pass, 41122 +github, level 0 with dict load, advanced one pass, 42173 github, level 1, advanced one pass, 142365 github, level 1 with dict, advanced one pass, 41266 github, level 1 with dict dms, advanced one pass, 41266 @@ -318,13 +318,13 @@ github, level 3, advanced github, level 3 with dict, advanced one pass, 41118 github, level 3 with dict dms, advanced one pass, 41118 github, level 3 with dict dds, advanced one pass, 41118 -github, level 3 with dict copy, advanced one pass, 41124 -github, level 3 with dict load, advanced one pass, 41847 +github, level 3 with dict copy, advanced one pass, 41122 +github, level 3 with dict load, advanced one pass, 42173 github, level 4, advanced one pass, 136199 github, level 4 with dict, advanced one pass, 41229 github, level 4 with dict dms, advanced one pass, 41229 github, level 4 with dict dds, advanced one pass, 41229 -github, level 4 with dict copy, advanced one pass, 41216 +github, level 4 with dict copy, advanced one pass, 41205 github, level 4 with dict load, advanced one pass, 41548 github, level 5 row 1, advanced one pass, 134584 github, level 5 row 1 with dict dms, advanced one pass, 38754 @@ -428,11 +428,11 @@ github.tar, level -3 with dict, advanced github.tar, level -1, advanced one pass, 42606 github.tar, level -1 with dict, advanced one pass, 41397 github.tar, level 0, advanced one pass, 38884 -github.tar, level 0 with dict, advanced one pass, 37995 +github.tar, level 0 with dict, advanced one pass, 38380 github.tar, level 0 with dict dms, advanced one pass, 38114 github.tar, level 0 with dict dds, advanced one pass, 38114 -github.tar, level 0 with dict copy, advanced one pass, 37995 -github.tar, level 0 with dict load, advanced one pass, 37956 +github.tar, level 0 with dict copy, advanced one pass, 38380 +github.tar, level 0 with dict load, advanced one pass, 38287 github.tar, level 1, advanced one pass, 39200 github.tar, level 1 with dict, advanced one pass, 38189 github.tar, level 1 with dict dms, advanced one pass, 38398 @@ -440,17 +440,17 @@ github.tar, level 1 with dict dds, advanced github.tar, level 1 with dict copy, advanced one pass, 38189 github.tar, level 1 with dict load, advanced one pass, 38393 github.tar, level 3, advanced one pass, 38884 -github.tar, level 3 with dict, advanced one pass, 37995 +github.tar, level 3 with dict, advanced one pass, 38380 github.tar, level 3 with dict dms, advanced one pass, 38114 github.tar, level 3 with dict dds, advanced one pass, 38114 -github.tar, level 3 with dict copy, advanced one pass, 37995 -github.tar, level 3 with dict load, advanced one pass, 37956 +github.tar, level 3 with dict copy, advanced one pass, 38380 +github.tar, level 3 with dict load, advanced one pass, 38287 github.tar, level 4, advanced one pass, 38880 -github.tar, level 4 with dict, advanced one pass, 37948 +github.tar, level 4 with dict, advanced one pass, 38293 github.tar, level 4 with dict dms, advanced one pass, 37995 github.tar, level 4 with dict dds, advanced one pass, 37995 -github.tar, level 4 with dict copy, advanced one pass, 37948 -github.tar, level 4 with dict load, advanced one pass, 37927 +github.tar, level 4 with dict copy, advanced one pass, 38293 +github.tar, level 4 with dict load, advanced one pass, 38298 github.tar, level 5 row 1, advanced one pass, 39651 github.tar, level 5 row 1 with dict dms, advanced one pass, 39043 github.tar, level 5 row 1 with dict dds, advanced one pass, 39069 @@ -534,7 +534,7 @@ github.tar, level 19 with dict dds, advanced github.tar, level 19 with dict copy, advanced one pass, 32701 github.tar, level 19 with dict load, advanced one pass, 32428 github.tar, no source size, advanced one pass, 38884 -github.tar, no source size with dict, advanced one pass, 37995 +github.tar, no source size with dict, advanced one pass, 38380 github.tar, long distance mode, advanced one pass, 40156 github.tar, multithreaded, advanced one pass, 38884 github.tar, multithreaded long distance mode, advanced one pass, 40139 @@ -624,8 +624,8 @@ github, level 0, advanced github, level 0 with dict, advanced one pass small out, 41118 github, level 0 with dict dms, advanced one pass small out, 41118 github, level 0 with dict dds, advanced one pass small out, 41118 -github, level 0 with dict copy, advanced one pass small out, 41124 -github, level 0 with dict load, advanced one pass small out, 41847 +github, level 0 with dict copy, advanced one pass small out, 41122 +github, level 0 with dict load, advanced one pass small out, 42173 github, level 1, advanced one pass small out, 142365 github, level 1 with dict, advanced one pass small out, 41266 github, level 1 with dict dms, advanced one pass small out, 41266 @@ -636,13 +636,13 @@ github, level 3, advanced github, level 3 with dict, advanced one pass small out, 41118 github, level 3 with dict dms, advanced one pass small out, 41118 github, level 3 with dict dds, advanced one pass small out, 41118 -github, level 3 with dict copy, advanced one pass small out, 41124 -github, level 3 with dict load, advanced one pass small out, 41847 +github, level 3 with dict copy, advanced one pass small out, 41122 +github, level 3 with dict load, advanced one pass small out, 42173 github, level 4, advanced one pass small out, 136199 github, level 4 with dict, advanced one pass small out, 41229 github, level 4 with dict dms, advanced one pass small out, 41229 github, level 4 with dict dds, advanced one pass small out, 41229 -github, level 4 with dict copy, advanced one pass small out, 41216 +github, level 4 with dict copy, advanced one pass small out, 41205 github, level 4 with dict load, advanced one pass small out, 41548 github, level 5 row 1, advanced one pass small out, 134584 github, level 5 row 1 with dict dms, advanced one pass small out, 38754 @@ -746,11 +746,11 @@ github.tar, level -3 with dict, advanced github.tar, level -1, advanced one pass small out, 42606 github.tar, level -1 with dict, advanced one pass small out, 41397 github.tar, level 0, advanced one pass small out, 38884 -github.tar, level 0 with dict, advanced one pass small out, 37995 +github.tar, level 0 with dict, advanced one pass small out, 38380 github.tar, level 0 with dict dms, advanced one pass small out, 38114 github.tar, level 0 with dict dds, advanced one pass small out, 38114 -github.tar, level 0 with dict copy, advanced one pass small out, 37995 -github.tar, level 0 with dict load, advanced one pass small out, 37956 +github.tar, level 0 with dict copy, advanced one pass small out, 38380 +github.tar, level 0 with dict load, advanced one pass small out, 38287 github.tar, level 1, advanced one pass small out, 39200 github.tar, level 1 with dict, advanced one pass small out, 38189 github.tar, level 1 with dict dms, advanced one pass small out, 38398 @@ -758,17 +758,17 @@ github.tar, level 1 with dict dds, advanced github.tar, level 1 with dict copy, advanced one pass small out, 38189 github.tar, level 1 with dict load, advanced one pass small out, 38393 github.tar, level 3, advanced one pass small out, 38884 -github.tar, level 3 with dict, advanced one pass small out, 37995 +github.tar, level 3 with dict, advanced one pass small out, 38380 github.tar, level 3 with dict dms, advanced one pass small out, 38114 github.tar, level 3 with dict dds, advanced one pass small out, 38114 -github.tar, level 3 with dict copy, advanced one pass small out, 37995 -github.tar, level 3 with dict load, advanced one pass small out, 37956 +github.tar, level 3 with dict copy, advanced one pass small out, 38380 +github.tar, level 3 with dict load, advanced one pass small out, 38287 github.tar, level 4, advanced one pass small out, 38880 -github.tar, level 4 with dict, advanced one pass small out, 37948 +github.tar, level 4 with dict, advanced one pass small out, 38293 github.tar, level 4 with dict dms, advanced one pass small out, 37995 github.tar, level 4 with dict dds, advanced one pass small out, 37995 -github.tar, level 4 with dict copy, advanced one pass small out, 37948 -github.tar, level 4 with dict load, advanced one pass small out, 37927 +github.tar, level 4 with dict copy, advanced one pass small out, 38293 +github.tar, level 4 with dict load, advanced one pass small out, 38298 github.tar, level 5 row 1, advanced one pass small out, 39651 github.tar, level 5 row 1 with dict dms, advanced one pass small out, 39043 github.tar, level 5 row 1 with dict dds, advanced one pass small out, 39069 @@ -852,7 +852,7 @@ github.tar, level 19 with dict dds, advanced github.tar, level 19 with dict copy, advanced one pass small out, 32701 github.tar, level 19 with dict load, advanced one pass small out, 32428 github.tar, no source size, advanced one pass small out, 38884 -github.tar, no source size with dict, advanced one pass small out, 37995 +github.tar, no source size with dict, advanced one pass small out, 38380 github.tar, long distance mode, advanced one pass small out, 40156 github.tar, multithreaded, advanced one pass small out, 38884 github.tar, multithreaded long distance mode, advanced one pass small out, 40139 @@ -901,10 +901,10 @@ silesia, multithreaded with advanced params, advanced silesia.tar, level -5, advanced streaming, 6853184 silesia.tar, level -3, advanced streaming, 6503455 silesia.tar, level -1, advanced streaming, 6175761 -silesia.tar, level 0, advanced streaming, 4846783 +silesia.tar, level 0, advanced streaming, 4839587 silesia.tar, level 1, advanced streaming, 5306719 -silesia.tar, level 3, advanced streaming, 4846783 -silesia.tar, level 4, advanced streaming, 4785332 +silesia.tar, level 3, advanced streaming, 4839587 +silesia.tar, level 4, advanced streaming, 4777299 silesia.tar, level 5 row 1, advanced streaming, 4664523 silesia.tar, level 5 row 2, advanced streaming, 4668292 silesia.tar, level 5, advanced streaming, 4664523 @@ -920,7 +920,7 @@ silesia.tar, level 12 row 2, advanced silesia.tar, level 13, advanced streaming, 4486652 silesia.tar, level 16, advanced streaming, 4358029 silesia.tar, level 19, advanced streaming, 4258228 -silesia.tar, no source size, advanced streaming, 4846779 +silesia.tar, no source size, advanced streaming, 4839583 silesia.tar, long distance mode, advanced streaming, 4825842 silesia.tar, multithreaded, advanced streaming, 4836000 silesia.tar, multithreaded long distance mode, advanced streaming, 4827826 @@ -928,7 +928,7 @@ silesia.tar, small window log, advanced silesia.tar, small hash log, advanced streaming, 6529503 silesia.tar, small chain log, advanced streaming, 4915956 silesia.tar, explicit params, advanced streaming, 4791739 -silesia.tar, uncompressed literals, advanced streaming, 5123274 +silesia.tar, uncompressed literals, advanced streaming, 5115568 silesia.tar, uncompressed literals optimal, advanced streaming, 4306968 silesia.tar, huffman literals, advanced streaming, 5323245 silesia.tar, multithreaded with advanced params, advanced streaming, 5116579 @@ -942,8 +942,8 @@ github, level 0, advanced github, level 0 with dict, advanced streaming, 41118 github, level 0 with dict dms, advanced streaming, 41118 github, level 0 with dict dds, advanced streaming, 41118 -github, level 0 with dict copy, advanced streaming, 41124 -github, level 0 with dict load, advanced streaming, 41847 +github, level 0 with dict copy, advanced streaming, 41122 +github, level 0 with dict load, advanced streaming, 42173 github, level 1, advanced streaming, 142365 github, level 1 with dict, advanced streaming, 41266 github, level 1 with dict dms, advanced streaming, 41266 @@ -954,13 +954,13 @@ github, level 3, advanced github, level 3 with dict, advanced streaming, 41118 github, level 3 with dict dms, advanced streaming, 41118 github, level 3 with dict dds, advanced streaming, 41118 -github, level 3 with dict copy, advanced streaming, 41124 -github, level 3 with dict load, advanced streaming, 41847 +github, level 3 with dict copy, advanced streaming, 41122 +github, level 3 with dict load, advanced streaming, 42173 github, level 4, advanced streaming, 136199 github, level 4 with dict, advanced streaming, 41229 github, level 4 with dict dms, advanced streaming, 41229 github, level 4 with dict dds, advanced streaming, 41229 -github, level 4 with dict copy, advanced streaming, 41216 +github, level 4 with dict copy, advanced streaming, 41205 github, level 4 with dict load, advanced streaming, 41548 github, level 5 row 1, advanced streaming, 134584 github, level 5 row 1 with dict dms, advanced streaming, 38754 @@ -1064,11 +1064,11 @@ github.tar, level -3 with dict, advanced github.tar, level -1, advanced streaming, 42687 github.tar, level -1 with dict, advanced streaming, 41486 github.tar, level 0, advanced streaming, 38884 -github.tar, level 0 with dict, advanced streaming, 37995 +github.tar, level 0 with dict, advanced streaming, 38380 github.tar, level 0 with dict dms, advanced streaming, 38114 github.tar, level 0 with dict dds, advanced streaming, 38114 -github.tar, level 0 with dict copy, advanced streaming, 37995 -github.tar, level 0 with dict load, advanced streaming, 37956 +github.tar, level 0 with dict copy, advanced streaming, 38380 +github.tar, level 0 with dict load, advanced streaming, 38287 github.tar, level 1, advanced streaming, 39346 github.tar, level 1 with dict, advanced streaming, 38251 github.tar, level 1 with dict dms, advanced streaming, 38557 @@ -1076,17 +1076,17 @@ github.tar, level 1 with dict dds, advanced github.tar, level 1 with dict copy, advanced streaming, 38251 github.tar, level 1 with dict load, advanced streaming, 38503 github.tar, level 3, advanced streaming, 38884 -github.tar, level 3 with dict, advanced streaming, 37995 +github.tar, level 3 with dict, advanced streaming, 38380 github.tar, level 3 with dict dms, advanced streaming, 38114 github.tar, level 3 with dict dds, advanced streaming, 38114 -github.tar, level 3 with dict copy, advanced streaming, 37995 -github.tar, level 3 with dict load, advanced streaming, 37956 +github.tar, level 3 with dict copy, advanced streaming, 38380 +github.tar, level 3 with dict load, advanced streaming, 38287 github.tar, level 4, advanced streaming, 38880 -github.tar, level 4 with dict, advanced streaming, 37948 +github.tar, level 4 with dict, advanced streaming, 38293 github.tar, level 4 with dict dms, advanced streaming, 37995 github.tar, level 4 with dict dds, advanced streaming, 37995 -github.tar, level 4 with dict copy, advanced streaming, 37948 -github.tar, level 4 with dict load, advanced streaming, 37927 +github.tar, level 4 with dict copy, advanced streaming, 38293 +github.tar, level 4 with dict load, advanced streaming, 38298 github.tar, level 5 row 1, advanced streaming, 39651 github.tar, level 5 row 1 with dict dms, advanced streaming, 39043 github.tar, level 5 row 1 with dict dds, advanced streaming, 39069 @@ -1203,10 +1203,10 @@ silesia, huffman literals, old stre silesia.tar, level -5, old streaming, 6853184 silesia.tar, level -3, old streaming, 6503455 silesia.tar, level -1, old streaming, 6175761 -silesia.tar, level 0, old streaming, 4846783 +silesia.tar, level 0, old streaming, 4839587 silesia.tar, level 1, old streaming, 5306719 -silesia.tar, level 3, old streaming, 4846783 -silesia.tar, level 4, old streaming, 4785332 +silesia.tar, level 3, old streaming, 4839587 +silesia.tar, level 4, old streaming, 4777299 silesia.tar, level 5, old streaming, 4664523 silesia.tar, level 6, old streaming, 4599420 silesia.tar, level 7, old streaming, 4565332 @@ -1214,8 +1214,8 @@ silesia.tar, level 9, old stre silesia.tar, level 13, old streaming, 4486652 silesia.tar, level 16, old streaming, 4358029 silesia.tar, level 19, old streaming, 4258228 -silesia.tar, no source size, old streaming, 4846779 -silesia.tar, uncompressed literals, old streaming, 4846783 +silesia.tar, no source size, old streaming, 4839583 +silesia.tar, uncompressed literals, old streaming, 4839587 silesia.tar, uncompressed literals optimal, old streaming, 4258228 silesia.tar, huffman literals, old streaming, 6175761 github, level -5, old streaming, 204407 @@ -1258,13 +1258,13 @@ github.tar, level -3 with dict, old stre github.tar, level -1, old streaming, 42687 github.tar, level -1 with dict, old streaming, 41486 github.tar, level 0, old streaming, 38884 -github.tar, level 0 with dict, old streaming, 37995 +github.tar, level 0 with dict, old streaming, 38380 github.tar, level 1, old streaming, 39346 github.tar, level 1 with dict, old streaming, 38251 github.tar, level 3, old streaming, 38884 -github.tar, level 3 with dict, old streaming, 37995 +github.tar, level 3 with dict, old streaming, 38380 github.tar, level 4, old streaming, 38880 -github.tar, level 4 with dict, old streaming, 37948 +github.tar, level 4 with dict, old streaming, 38293 github.tar, level 5, old streaming, 39651 github.tar, level 5 with dict, old streaming, 39145 github.tar, level 6, old streaming, 39282 @@ -1313,10 +1313,10 @@ silesia, multithreaded with advanced params, old stre silesia.tar, level -5, old streaming advanced, 6853184 silesia.tar, level -3, old streaming advanced, 6503455 silesia.tar, level -1, old streaming advanced, 6175761 -silesia.tar, level 0, old streaming advanced, 4846783 +silesia.tar, level 0, old streaming advanced, 4839587 silesia.tar, level 1, old streaming advanced, 5306719 -silesia.tar, level 3, old streaming advanced, 4846783 -silesia.tar, level 4, old streaming advanced, 4785332 +silesia.tar, level 3, old streaming advanced, 4839587 +silesia.tar, level 4, old streaming advanced, 4777299 silesia.tar, level 5, old streaming advanced, 4664523 silesia.tar, level 6, old streaming advanced, 4599420 silesia.tar, level 7, old streaming advanced, 4565332 @@ -1324,18 +1324,18 @@ silesia.tar, level 9, old stre silesia.tar, level 13, old streaming advanced, 4486652 silesia.tar, level 16, old streaming advanced, 4358029 silesia.tar, level 19, old streaming advanced, 4258228 -silesia.tar, no source size, old streaming advanced, 4846779 -silesia.tar, long distance mode, old streaming advanced, 4846783 -silesia.tar, multithreaded, old streaming advanced, 4846783 -silesia.tar, multithreaded long distance mode, old streaming advanced, 4846783 +silesia.tar, no source size, old streaming advanced, 4839583 +silesia.tar, long distance mode, old streaming advanced, 4839587 +silesia.tar, multithreaded, old streaming advanced, 4839587 +silesia.tar, multithreaded long distance mode, old streaming advanced, 4839587 silesia.tar, small window log, old streaming advanced, 7117027 silesia.tar, small hash log, old streaming advanced, 6529503 silesia.tar, small chain log, old streaming advanced, 4915956 silesia.tar, explicit params, old streaming advanced, 4791739 -silesia.tar, uncompressed literals, old streaming advanced, 4846783 +silesia.tar, uncompressed literals, old streaming advanced, 4839587 silesia.tar, uncompressed literals optimal, old streaming advanced, 4258228 silesia.tar, huffman literals, old streaming advanced, 6175761 -silesia.tar, multithreaded with advanced params, old streaming advanced, 4846783 +silesia.tar, multithreaded with advanced params, old streaming advanced, 4839587 github, level -5, old streaming advanced, 213265 github, level -5 with dict, old streaming advanced, 46708 github, level -3, old streaming advanced, 196126 @@ -1384,13 +1384,13 @@ github.tar, level -3 with dict, old stre github.tar, level -1, old streaming advanced, 42687 github.tar, level -1 with dict, old streaming advanced, 41762 github.tar, level 0, old streaming advanced, 38884 -github.tar, level 0 with dict, old streaming advanced, 38013 +github.tar, level 0 with dict, old streaming advanced, 38442 github.tar, level 1, old streaming advanced, 39346 github.tar, level 1 with dict, old streaming advanced, 38507 github.tar, level 3, old streaming advanced, 38884 -github.tar, level 3 with dict, old streaming advanced, 38013 +github.tar, level 3 with dict, old streaming advanced, 38442 github.tar, level 4, old streaming advanced, 38880 -github.tar, level 4 with dict, old streaming advanced, 38063 +github.tar, level 4 with dict, old streaming advanced, 38473 github.tar, level 5, old streaming advanced, 39651 github.tar, level 5 with dict, old streaming advanced, 39018 github.tar, level 6, old streaming advanced, 39282 @@ -1436,10 +1436,10 @@ github, no source size with dict, old stre github.tar, level -5 with dict, old streaming cdict, 51407 github.tar, level -3 with dict, old streaming cdict, 45254 github.tar, level -1 with dict, old streaming cdict, 41973 -github.tar, level 0 with dict, old streaming cdict, 37956 +github.tar, level 0 with dict, old streaming cdict, 38287 github.tar, level 1 with dict, old streaming cdict, 38503 -github.tar, level 3 with dict, old streaming cdict, 37956 -github.tar, level 4 with dict, old streaming cdict, 37927 +github.tar, level 3 with dict, old streaming cdict, 38287 +github.tar, level 4 with dict, old streaming cdict, 38298 github.tar, level 5 with dict, old streaming cdict, 39000 github.tar, level 6 with dict, old streaming cdict, 38647 github.tar, level 7 with dict, old streaming cdict, 37286 @@ -1466,10 +1466,10 @@ github, no source size with dict, old stre github.tar, level -5 with dict, old streaming advanced cdict, 50907 github.tar, level -3 with dict, old streaming advanced cdict, 45032 github.tar, level -1 with dict, old streaming advanced cdict, 41589 -github.tar, level 0 with dict, old streaming advanced cdict, 38013 +github.tar, level 0 with dict, old streaming advanced cdict, 38442 github.tar, level 1 with dict, old streaming advanced cdict, 38294 -github.tar, level 3 with dict, old streaming advanced cdict, 38013 -github.tar, level 4 with dict, old streaming advanced cdict, 38063 +github.tar, level 3 with dict, old streaming advanced cdict, 38442 +github.tar, level 4 with dict, old streaming advanced cdict, 38473 github.tar, level 5 with dict, old streaming advanced cdict, 39018 github.tar, level 6 with dict, old streaming advanced cdict, 38635 github.tar, level 7 with dict, old streaming advanced cdict, 37264