diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index 8d108c72ece..a344a2bb25d 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -1241,15 +1241,8 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c size_t prevOffset0 = seqState->prevOffset[0]; size_t prevOffset1 = seqState->prevOffset[1]; size_t prevOffset2 = seqState->prevOffset[2]; - /* - * ZSTD_seqSymbol is a 64 bits wide structure. - * It can be loaded in one operation - * and its fields extracted by simply shifting or bit-extracting on aarch64. - * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh - * operations that cause performance drop. This can be avoided by using this - * ZSTD_memcpy hack. - */ -# if defined(__GNUC__) && !defined(__clang__) + /* Encourage a single 64-bit load per symbol, followed by field extraction. */ +# if defined(__GNUC__) ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS; ZSTD_seqSymbol* const llDInfo = &llDInfoS; ZSTD_seqSymbol* const mlDInfo = &mlDInfoS; @@ -1323,26 +1316,42 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c seq.offset = offset; } - if (mlBits > 0) - seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); - - if (UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - - /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + /* Most symbols need no additional literal-length bits. Keep their + * match-length read separate from the uncommon two-length case. */ ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - - if (llBits > 0) - seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + if (UNLIKELY(llBits != 0)) { + if (mlBits != 0) + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits); + if (UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits); + } else if (mlBits != 0) { + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits); + if (UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + } else if (UNLIKELY(ofBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) { + BIT_reloadDStream(&seqState->DStream); + } DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); if (!isLastSeq) { /* Don't update FSE state for last sequence. */ - ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + /* Compute each bit position from the same starting position to + * avoid serial updates of bitsConsumed. */ + { U32 const consumed = seqState->DStream.bitsConsumed; + U32 const llmlBits = llnbBits + mlnbBits; + U32 const allFseBits = llmlBits + ofnbBits; + BitContainerType const container = seqState->DStream.bitContainer; + seqState->stateLL.state = llNext + BIT_getMiddleBits( + container, 64 - (consumed + llnbBits), llnbBits); + seqState->stateML.state = mlNext + BIT_getMiddleBits( + container, 64 - (consumed + llmlBits), mlnbBits); + seqState->stateOffb.state = ofNext + BIT_getMiddleBits( + container, 64 - (consumed + allFseBits), ofnbBits); + seqState->DStream.bitsConsumed = consumed + allFseBits; + } BIT_reloadDStream(&seqState->DStream); } }