diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index 56e1f9eda28..702b20680b7 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -17,6 +17,7 @@ #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ #include "../common/compiler.h" /* prefetch */ #include "../common/mem.h" /* low level memory routines */ +#include "../common/bitstream.h" /* bitstream */ #include #define FSE_STATIC_LINKING_ONLY #include "../common/fse.h" @@ -912,45 +913,49 @@ static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, size_t length) FORCE_NOINLINE ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_execSequenceEnd(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) + BYTE* const oend, size_t litLength, size_t matchLength, size_t offset, + const BYTE* litPtr, const BYTE* const litLimit, + const ZSTD_DCtx* dctx, + const int hasExtDict) { - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = oLitEnd - sequence.offset; + const BYTE* const prefixStart = (const BYTE*)dctx->prefixStart; + const BYTE* const virtualStart = (const BYTE*)dctx->virtualStart; + const BYTE* const dictEnd = (const BYTE*)dctx->dictEnd; + BYTE* const oLitEnd = op + litLength; + size_t const sequenceLength = litLength + matchLength; + const BYTE* match = oLitEnd - offset; BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* bounds checks : careful of address space overflow in 32-bit mode */ RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); - RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + RETURN_ERROR_IF(litLength > (size_t)(litLimit - litPtr), corruption_detected, "try to read beyond literal buffer"); assert(op < op + sequenceLength); assert(oLitEnd < op + sequenceLength); /* copy literals */ - ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + ZSTD_safecopy(op, oend_w, litPtr, litLength, ZSTD_no_overlap); op = oLitEnd; - *litPtr = iLitEnd; /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + if (hasExtDict && offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix */ - RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + RETURN_ERROR_IF(offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); match = dictEnd - (prefixStart - match); - if (match + sequence.matchLength <= dictEnd) { - ZSTD_memmove(oLitEnd, match, sequence.matchLength); + if (match + matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, matchLength); return sequenceLength; } /* span extDict & currentPrefixSegment */ { size_t const length1 = (size_t)(dictEnd - match); ZSTD_memmove(oLitEnd, match, length1); op = oLitEnd + length1; - sequence.matchLength -= length1; + matchLength -= length1; match = prefixStart; } + } else { + RETURN_ERROR_IF(UNLIKELY(offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); } - ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + ZSTD_safecopy(op, oend_w, match, matchLength, ZSTD_overlap_src_before_dst); return sequenceLength; } @@ -960,62 +965,70 @@ size_t ZSTD_execSequenceEnd(BYTE* op, FORCE_NOINLINE ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op, - BYTE* const oend, const BYTE* const oend_w, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) + BYTE* const oend, const BYTE* const oend_w, size_t litLength, size_t matchLength, size_t offset, + const BYTE* litPtr, const BYTE* const litLimit, + const ZSTD_DCtx* dctx, + const int hasExtDict) { - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = oLitEnd - sequence.offset; + const BYTE* const prefixStart = (const BYTE*)dctx->prefixStart; + const BYTE* const virtualStart = (const BYTE*)dctx->virtualStart; + const BYTE* const dictEnd = (const BYTE*)dctx->dictEnd; + BYTE* const oLitEnd = op + litLength; + size_t const sequenceLength = litLength + matchLength; + const BYTE* match = oLitEnd - offset; /* bounds checks : careful of address space overflow in 32-bit mode */ RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); - RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + RETURN_ERROR_IF(litLength > (size_t)(litLimit - litPtr), corruption_detected, "try to read beyond literal buffer"); assert(op < op + sequenceLength); assert(oLitEnd < op + sequenceLength); /* copy literals */ - RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); - ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength); + RETURN_ERROR_IF(op > litPtr && op < litPtr + litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, litLength); op = oLitEnd; - *litPtr = iLitEnd; /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + if (hasExtDict && offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix */ - RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + RETURN_ERROR_IF(offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); match = dictEnd - (prefixStart - match); - if (match + sequence.matchLength <= dictEnd) { - ZSTD_memmove(oLitEnd, match, sequence.matchLength); + if (match + matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, matchLength); return sequenceLength; } /* span extDict & currentPrefixSegment */ { size_t const length1 = (size_t)(dictEnd - match); ZSTD_memmove(oLitEnd, match, length1); op = oLitEnd + length1; - sequence.matchLength -= length1; + matchLength -= length1; match = prefixStart; } + } else { + RETURN_ERROR_IF(UNLIKELY(offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); } - ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + ZSTD_safecopy(op, oend_w, match, matchLength, ZSTD_overlap_src_before_dst); return sequenceLength; } HINT_INLINE ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_execSequence(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) + BYTE* const oend, size_t litLength, size_t matchLength, size_t offset, + const BYTE* litPtr, const BYTE* const litLimit, + const ZSTD_DCtx* dctx, + const int hasExtDict) { - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const prefixStart = (const BYTE*)dctx->prefixStart; + const BYTE* const virtualStart = (const BYTE*)dctx->virtualStart; + const BYTE* const dictEnd = (const BYTE*)dctx->dictEnd; + BYTE* const oLitEnd = op + litLength; + size_t const sequenceLength = litLength + matchLength; BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */ - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = oLitEnd - sequence.offset; + const BYTE* const iLitEnd = litPtr + litLength; + const BYTE* match = oLitEnd - offset; assert(op != NULL /* Precondition */); assert(oend_w < oend /* No underflow */); @@ -1033,7 +1046,7 @@ size_t ZSTD_execSequence(BYTE* op, iLitEnd > litLimit || oMatchEnd > oend_w || (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) - return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + return ZSTD_execSequenceEnd(op, oend, litLength, matchLength, offset, litPtr, litLimit, dctx, hasExtDict); /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ assert(op <= oLitEnd /* No overflow */); @@ -1048,56 +1061,57 @@ size_t ZSTD_execSequence(BYTE* op, * We likely don't need the full 32-byte wildcopy. */ assert(WILDCOPY_OVERLENGTH >= 16); - ZSTD_copy16(op, (*litPtr)); - if (UNLIKELY(sequence.litLength > 16)) { - ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); + ZSTD_copy16(op, litPtr); + if (UNLIKELY(litLength > 16)) { + ZSTD_wildcopy(op + 16, litPtr + 16, litLength - 16, ZSTD_no_overlap); } op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ /* Copy Match */ - if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + if (hasExtDict && offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix -> go into extDict */ - RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + RETURN_ERROR_IF(UNLIKELY(offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); match = dictEnd + (match - prefixStart); - if (match + sequence.matchLength <= dictEnd) { - ZSTD_memmove(oLitEnd, match, sequence.matchLength); + if (match + matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, matchLength); return sequenceLength; } /* span extDict & currentPrefixSegment */ { size_t const length1 = (size_t)(dictEnd - match); ZSTD_memmove(oLitEnd, match, length1); op = oLitEnd + length1; - sequence.matchLength -= length1; + matchLength -= length1; match = prefixStart; } + } else { + RETURN_ERROR_IF(UNLIKELY(offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); } /* Match within prefix of 1 or more bytes */ assert(op <= oMatchEnd); assert(oMatchEnd <= oend_w); assert(match >= prefixStart); - assert(sequence.matchLength >= 1); + assert(matchLength >= 1); /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy * without overlap checking. */ - if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + if (LIKELY(offset >= WILDCOPY_VECLEN)) { /* We bet on a full wildcopy for matches, since we expect matches to be * longer than literals (in general). In silesia, ~10% of matches are longer * than 16 bytes. */ - ZSTD_wildcopy(op, match, sequence.matchLength, ZSTD_no_overlap); + ZSTD_wildcopy(op, match, matchLength, ZSTD_no_overlap); return sequenceLength; } - assert(sequence.offset < WILDCOPY_VECLEN); + assert(offset < WILDCOPY_VECLEN); /* Copy 8 bytes and spread the offset to be >= 8. */ - ZSTD_overlapCopy8(&op, &match, sequence.offset); + ZSTD_overlapCopy8(&op, &match, offset); /* If the match length is > 8 bytes, then continue with the wildcopy. */ - if (sequence.matchLength > 8) { + if (matchLength > 8) { assert(op < oMatchEnd); - ZSTD_wildcopy(op, match, sequence.matchLength - 8, ZSTD_overlap_src_before_dst); + ZSTD_wildcopy(op, match, matchLength - 8, ZSTD_overlap_src_before_dst); } return sequenceLength; } @@ -1105,15 +1119,19 @@ size_t ZSTD_execSequence(BYTE* op, HINT_INLINE ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, - BYTE* const oend, const BYTE* const oend_w, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) + BYTE* const oend, const BYTE* const oend_w, size_t litLength, size_t matchLength, size_t offset, + const BYTE* litPtr, const BYTE* const litLimit, + const ZSTD_DCtx* dctx, + const int hasExtDict) { - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const prefixStart = (const BYTE*)dctx->prefixStart; + const BYTE* const virtualStart = (const BYTE*)dctx->virtualStart; + const BYTE* const dictEnd = (const BYTE*)dctx->dictEnd; + BYTE* const oLitEnd = op + litLength; + size_t const sequenceLength = litLength + matchLength; BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = oLitEnd - sequence.offset; + const BYTE* const iLitEnd = litPtr + litLength; + const BYTE* match = oLitEnd - offset; assert(op != NULL /* Precondition */); assert(oend_w < oend /* No underflow */); @@ -1126,7 +1144,7 @@ size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, iLitEnd > litLimit || oMatchEnd > oend_w || (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) - return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, litLength, matchLength, offset, litPtr, litLimit, dctx, hasExtDict); /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ assert(op <= oLitEnd /* No overflow */); @@ -1141,55 +1159,56 @@ size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, * We likely don't need the full 32-byte wildcopy. */ assert(WILDCOPY_OVERLENGTH >= 16); - ZSTD_copy16(op, (*litPtr)); - if (UNLIKELY(sequence.litLength > 16)) { - ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); + ZSTD_copy16(op, litPtr); + if (UNLIKELY(litLength > 16)) { + ZSTD_wildcopy(op+16, litPtr+16, litLength-16, ZSTD_no_overlap); } op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ /* Copy Match */ - if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + if (hasExtDict && offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix -> go into extDict */ - RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + RETURN_ERROR_IF(UNLIKELY(offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); match = dictEnd + (match - prefixStart); - if (match + sequence.matchLength <= dictEnd) { - ZSTD_memmove(oLitEnd, match, sequence.matchLength); + if (match + matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, matchLength); return sequenceLength; } /* span extDict & currentPrefixSegment */ { size_t const length1 = (size_t)(dictEnd - match); ZSTD_memmove(oLitEnd, match, length1); op = oLitEnd + length1; - sequence.matchLength -= length1; + matchLength -= length1; match = prefixStart; - } } + } } else { + RETURN_ERROR_IF(UNLIKELY(offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + } /* Match within prefix of 1 or more bytes */ assert(op <= oMatchEnd); assert(oMatchEnd <= oend_w); assert(match >= prefixStart); - assert(sequence.matchLength >= 1); + assert(matchLength >= 1); /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy * without overlap checking. */ - if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + if (LIKELY(offset >= WILDCOPY_VECLEN)) { /* We bet on a full wildcopy for matches, since we expect matches to be * longer than literals (in general). In silesia, ~10% of matches are longer * than 16 bytes. */ - ZSTD_wildcopy(op, match, sequence.matchLength, ZSTD_no_overlap); + ZSTD_wildcopy(op, match, matchLength, ZSTD_no_overlap); return sequenceLength; } - assert(sequence.offset < WILDCOPY_VECLEN); + assert(offset < WILDCOPY_VECLEN); /* Copy 8 bytes and spread the offset to be >= 8. */ - ZSTD_overlapCopy8(&op, &match, sequence.offset); + ZSTD_overlapCopy8(&op, &match, offset); /* If the match length is > 8 bytes, then continue with the wildcopy. */ - if (sequence.matchLength > 8) { + if (matchLength > 8) { assert(op < oMatchEnd); - ZSTD_wildcopy(op, match, sequence.matchLength-8, ZSTD_overlap_src_before_dst); + ZSTD_wildcopy(op, match, matchLength-8, ZSTD_overlap_src_before_dst); } return sequenceLength; } @@ -1226,6 +1245,46 @@ ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; +FORCE_INLINE_TEMPLATE size_t ZSTD_decodeSmallOffset( + BIT_DStream_t* DStream, size_t* prevOffset0, size_t* prevOffset1, + size_t* prevOffset2, U32 ll0, U32 ofBase, BYTE ofBits) { + /* ofBits is either 0 or 1. + * Looking ahead 1 bit is sufficient, and ofBits acts as both a bit mask and bit count. */ + assert(ofBits <= 1); + + if (LIKELY(ofBits == 0)) { + size_t const offset = ll0 ? *prevOffset1 : *prevOffset0; + if (ll0) { + *prevOffset1 = *prevOffset0; + } + *prevOffset0 = offset; + return offset; + } + + size_t const rawBits = BIT_lookBitsFast(DStream, 1); + BIT_skipBits(DStream, 1); + + size_t const offset_eq1 = ofBase + ll0 + rawBits; + + size_t temp = *prevOffset0; + if (offset_eq1 >= 2) temp = *prevOffset2; + if (offset_eq1 == 3) temp = *prevOffset0 - 1; + if (offset_eq1 == 1) temp = *prevOffset1; + + temp -= !temp; /* A previous offset of 0 is only possible if input is + * corrupted. In this case, temp becomes (size_t)-1, which is + * an invalid offset, and will be detected as such. + */ + size_t new_prev2 = *prevOffset1; + if (offset_eq1 == 1) new_prev2 = *prevOffset2; + + *prevOffset2 = new_prev2; + *prevOffset1 = *prevOffset0; + *prevOffset0 = temp; + + return temp; +} + /** * ZSTD_decodeSequence(): * @p longOffsets : tells the decoder to reload more bit while decoding large offsets @@ -1297,28 +1356,10 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c prevOffset0 = offset; } else { U32 const ll0 = (llDInfo->baseValue == 0); - if (LIKELY((ofBits == 0))) { - if (ll0) { - offset = prevOffset1; - prevOffset1 = prevOffset0; - prevOffset0 = offset; - } else { - offset = prevOffset0; - } - } else { - offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); - { size_t temp = (offset == 1) ? prevOffset1 - : (offset == 3) ? prevOffset0 - 1 - : (offset >= 2) ? prevOffset2 - : prevOffset0; - /* 0 is not valid: input corrupted => force offset to -1 => - * corruption detected at execSequence. - */ - temp -= !temp; - prevOffset2 = (offset == 1) ? prevOffset2 : prevOffset1; - prevOffset1 = prevOffset0; - prevOffset0 = offset = temp; - } } } + offset = ZSTD_decodeSmallOffset( + &seqState->DStream, &prevOffset0, &prevOffset1, + &prevOffset2, ll0, ofBase, ofBits); + } seq.offset = offset; } @@ -1397,19 +1438,19 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset; } else { + size_t prevOffset0 = seqState->prevOffset[0]; + size_t prevOffset1 = seqState->prevOffset[1]; + size_t prevOffset2 = seqState->prevOffset[2]; + U32 const ll0 = (llDInfo->baseValue == 0); - if (LIKELY((ofBits == 0))) { - offset = seqState->prevOffset[ll0]; - seqState->prevOffset[1] = seqState->prevOffset[!ll0]; - seqState->prevOffset[0] = offset; - } else { - offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); - { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp -= !temp; /* 0 is not valid: input corrupted => force offset to -1 => corruption detected at execSequence */ - if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } } } + offset = ZSTD_decodeSmallOffset(&seqState->DStream, &prevOffset0, + &prevOffset1, &prevOffset2, ll0, + ofBase, ofBits); + + seqState->prevOffset[2] = prevOffset2; + seqState->prevOffset[1] = prevOffset1; + seqState->prevOffset[0] = prevOffset0; + } seq.offset = offset; } @@ -1504,7 +1545,8 @@ DONT_VECTORIZE ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) + const ZSTD_longOffset_e isLongOffset, + const int hasExtDict) { BYTE* const ostart = (BYTE*)dst; BYTE* const oend = (BYTE*)ZSTD_maybeNullPtrAdd(ostart, (ptrdiff_t)maxDstSize); @@ -1512,8 +1554,11 @@ ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, const BYTE* litPtr = dctx->litPtr; const BYTE* litBufferEnd = dctx->litBufferEnd; const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + (void)prefixStart; const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); + (void)vBase; const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + (void)dictEnd; DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer (%i seqs)", nbSeq); /* Literals are split between internal buffer & output buffer */ @@ -1600,7 +1645,8 @@ ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, for ( ; nbSeq; nbSeq--) { sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); if (litPtr + sequence.litLength > dctx->litBufferEnd) break; - { size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); + { size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence.litLength, sequence.matchLength, sequence.offset, litPtr, litBufferEnd, dctx, hasExtDict); + litPtr += sequence.litLength; #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) assert(!ZSTD_isError(oneSeqSize)); ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); @@ -1626,7 +1672,8 @@ ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, litPtr = dctx->litExtraBuffer; litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; dctx->litBufferLocation = ZSTD_not_in_dst; - { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence.litLength, sequence.matchLength, sequence.offset, litPtr, litBufferEnd, dctx, hasExtDict); + litPtr += sequence.litLength; #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) assert(!ZSTD_isError(oneSeqSize)); ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); @@ -1662,7 +1709,8 @@ ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, for ( ; nbSeq ; nbSeq--) { seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence.litLength, sequence.matchLength, sequence.offset, litPtr, litBufferEnd, dctx, hasExtDict); + litPtr += sequence.litLength; #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) assert(!ZSTD_isError(oneSeqSize)); ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); @@ -1715,7 +1763,8 @@ DONT_VECTORIZE ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) + const ZSTD_longOffset_e isLongOffset, + const int hasExtDict) { BYTE* const ostart = (BYTE*)dst; BYTE* const oend = (dctx->litBufferLocation == ZSTD_not_in_dst) ? @@ -1726,7 +1775,9 @@ ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); + (void)vBase; const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + (void)dictEnd; DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq); /* Regen sequences */ @@ -1756,9 +1807,32 @@ ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, # endif #endif + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; for ( ; nbSeq ; nbSeq--) { seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); + size_t oneSeqSize; + if (LIKELY( + (litPtr + sequence.litLength <= litEnd) && + (op + sequence.litLength + sequence.matchLength <= oend_w) && + (sequence.offset >= WILDCOPY_VECLEN) && + (sequence.offset <= (size_t)((op + sequence.litLength) - prefixStart)) && + (!MEM_32bits() || (size_t)(oend - op) >= sequence.litLength + sequence.matchLength + WILDCOPY_OVERLENGTH) + )) { + BYTE* const oLitEnd = op + sequence.litLength; + const BYTE* const match = oLitEnd - sequence.offset; + + ZSTD_copy16(op, litPtr); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op + 16, litPtr + 16, sequence.litLength - 16, ZSTD_no_overlap); + } + litPtr += sequence.litLength; + + ZSTD_wildcopy(oLitEnd, match, sequence.matchLength, ZSTD_no_overlap); + oneSeqSize = sequence.litLength + sequence.matchLength; + } else { + oneSeqSize = ZSTD_execSequence(op, oend, sequence.litLength, sequence.matchLength, sequence.offset, litPtr, litEnd, dctx, hasExtDict); + litPtr += sequence.litLength; + } #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) assert(!ZSTD_isError(oneSeqSize)); ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); @@ -1795,7 +1869,11 @@ ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + int const hasExtDict = (dctx->prefixStart != dctx->virtualStart); + if (hasExtDict) + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, 1); + else + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, 0); } static size_t @@ -1804,7 +1882,11 @@ ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx, const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + int const hasExtDict = (dctx->prefixStart != dctx->virtualStart); + if (hasExtDict) + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, 1); + else + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, 0); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ @@ -1834,7 +1916,8 @@ ZSTD_decompressSequencesLong_body( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) + const ZSTD_longOffset_e isLongOffset, + const int hasExtDict) { BYTE* const ostart = (BYTE*)dst; BYTE* const oend = (dctx->litBufferLocation == ZSTD_in_dst) ? @@ -1844,8 +1927,11 @@ ZSTD_decompressSequencesLong_body( const BYTE* litPtr = dctx->litPtr; const BYTE* litBufferEnd = dctx->litBufferEnd; const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + (void)prefixStart; const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); + (void)dictStart; const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + (void)dictEnd; /* Regen sequences */ if (nbSeq) { @@ -1892,7 +1978,8 @@ ZSTD_decompressSequencesLong_body( litPtr = dctx->litExtraBuffer; litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; dctx->litBufferLocation = ZSTD_not_in_dst; - { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].matchLength, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].offset, litPtr, litBufferEnd, dctx, hasExtDict); + litPtr += sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength; #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) assert(!ZSTD_isError(oneSeqSize)); ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); @@ -1907,8 +1994,9 @@ ZSTD_decompressSequencesLong_body( { /* lit buffer is either wholly contained in first or second split, or not split at all*/ size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? - ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : - ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].matchLength, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].offset, litPtr, litBufferEnd, dctx, hasExtDict) : + ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].matchLength, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].offset, litPtr, litBufferEnd, dctx, hasExtDict); + litPtr += sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength; #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) assert(!ZSTD_isError(oneSeqSize)); ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); @@ -1938,7 +2026,8 @@ ZSTD_decompressSequencesLong_body( litPtr = dctx->litExtraBuffer; litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; dctx->litBufferLocation = ZSTD_not_in_dst; - { size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence->litLength, sequence->matchLength, sequence->offset, litPtr, litBufferEnd, dctx, hasExtDict); + litPtr += sequence->litLength; #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) assert(!ZSTD_isError(oneSeqSize)); ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); @@ -1950,8 +2039,9 @@ ZSTD_decompressSequencesLong_body( else { size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? - ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : - ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, sequence->litLength, sequence->matchLength, sequence->offset, litPtr, litBufferEnd, dctx, hasExtDict) : + ZSTD_execSequence(op, oend, sequence->litLength, sequence->matchLength, sequence->offset, litPtr, litBufferEnd, dctx, hasExtDict); + litPtr += sequence->litLength; #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) assert(!ZSTD_isError(oneSeqSize)); ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); @@ -1995,7 +2085,11 @@ ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + int const hasExtDict = (dctx->prefixStart != dctx->virtualStart); + if (hasExtDict) + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, 1); + else + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, 0); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ @@ -2011,7 +2105,11 @@ ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + int const hasExtDict = (dctx->prefixStart != dctx->virtualStart); + if (hasExtDict) + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, 1); + else + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, 0); } static BMI2_TARGET_ATTRIBUTE size_t DONT_VECTORIZE @@ -2020,7 +2118,11 @@ ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx, const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + int const hasExtDict = (dctx->prefixStart != dctx->virtualStart); + if (hasExtDict) + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, 1); + else + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, 0); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ @@ -2031,7 +2133,11 @@ ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + int const hasExtDict = (dctx->prefixStart != dctx->virtualStart); + if (hasExtDict) + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, 1); + else + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, 0); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */