From cc101959870fc5a37e82099611c193486845d1dc Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 7 Sep 2026 12:24:20 -0700 Subject: [PATCH] decompress: sink nextState/nbBits loads in ZSTD_decodeSequence The generic path loads all twelve fields of the three ZSTD_seqSymbol entries up front, but the six nextState/nbBits values are not consumed until the final FSE state update. Keeping them live across the whole offset, matchLength and litLength decode costs six registers, and gcc 11.5.0 -O3 loses that fight: two of the three nextState values are spilled and reloaded per sequence in the hot loop of ZSTD_decompressSequences_bmi2(). mov %bx,0x20(%rsp) ... mov %bx,0x28(%rsp) ... movzwl 0x20(%rsp),%ecx ... movzwl 0x28(%rsp),%ecx clang 22.1.3 is worse, not better: it materialises all six at the top of the block and spills four of them immediately, so each makes a memory -> register -> stack -> register round trip before it is used. d3af: movzwl (%rsi,%rdi,8),%r13d d3b4: mov %r13,0x178(%rsp) Read them at the point of use instead, turning each spill/reload pair back into an L1 hit on a line already touched for nbAdditionalBits and baseValue. That removes all of the nextState spill/reload pairs, in every ZSTD_decompressSequences* instantiation; for clang it holds at -O2, -O3, ThinLTO and -march=native alike. Measured on an AMD EPYC 9D85 over three 32 MiB corpora -- lorem ipsum text, JSON log lines, C/Python source -- compressed at level 3 and decompressed in memory with a reused DCtx. Throughput is the mean over 16 code layouts per revision, produced by linking a padding object ahead of the library: layout alone moves this loop by up to 4%, so a single build pair cannot resolve the effect. 128 KiB payload before after gcc text 1230.1 1309.0 +6.4% source 1435.1 1476.8 +2.9% JSON 1777.3 1786.3 +0.5% clang text 958.0 990.4 +3.4% source 1205.1 1229.2 +2.0% JSON 1537.4 1533.1 -0.3% 16 KiB payloads track the same pattern, +1.1% to +4.3% for gcc and -0.8% to +2.4% for clang. 4 KiB payloads are flat, as expected: there the per-block header decode dominates over the sequence loop. Signed-off-by: Usama Arif --- lib/decompress/zstd_decompress_block.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index 56e1f9eda28..7f02545a4ea 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -1360,13 +1360,6 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c BYTE const ofBits = ofDInfo->nbAdditionalBits; BYTE const totalBits = llBits+mlBits+ofBits; - U16 const llNext = llDInfo->nextState; - U16 const mlNext = mlDInfo->nextState; - U16 const ofNext = ofDInfo->nextState; - U32 const llnbBits = llDInfo->nbBits; - U32 const mlnbBits = mlDInfo->nbBits; - U32 const ofnbBits = ofDInfo->nbBits; - assert(llBits <= MaxLLBits); assert(mlBits <= MaxMLBits); assert(ofBits <= MaxOff); @@ -1434,10 +1427,13 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c if (!isLastSeq) { /* Don't update FSE state for last sequence. */ - ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, + llDInfo->nextState, llDInfo->nbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, + mlDInfo->nextState, mlDInfo->nbBits); /* <= 9 bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, + ofDInfo->nextState, ofDInfo->nbBits); /* <= 8 bits */ BIT_reloadDStream(&seqState->DStream); } }