From e9325265c1a8bf2df5ad116369b22883f7169c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= Date: Sat, 3 Oct 2026 16:54:16 +0200 Subject: [PATCH 1/2] AArch64: Improve Clang sequence decoding Assisted-by: OpenAI Codex --- lib/decompress/zstd_decompress_block.c | 32 ++++++++++++++++++-------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index 8d108c72ece..3f30191ba27 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -1241,15 +1241,8 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c size_t prevOffset0 = seqState->prevOffset[0]; size_t prevOffset1 = seqState->prevOffset[1]; size_t prevOffset2 = seqState->prevOffset[2]; - /* - * ZSTD_seqSymbol is a 64 bits wide structure. - * It can be loaded in one operation - * and its fields extracted by simply shifting or bit-extracting on aarch64. - * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh - * operations that cause performance drop. This can be avoided by using this - * ZSTD_memcpy hack. - */ -# if defined(__GNUC__) && !defined(__clang__) + /* Encourage a single 64-bit load per symbol, followed by field extraction. */ +# if defined(__GNUC__) ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS; ZSTD_seqSymbol* const llDInfo = &llDInfoS; ZSTD_seqSymbol* const mlDInfo = &mlDInfoS; @@ -1323,6 +1316,25 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c seq.offset = offset; } +# if defined(__clang__) + /* Most symbols need no additional literal-length bits. Keep their + * match-length read separate from the uncommon two-length case. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + if (UNLIKELY(llBits != 0)) { + if (mlBits != 0) + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits); + if (UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits); + } else if (mlBits != 0) { + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits); + if (UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + } else if (UNLIKELY(ofBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) { + BIT_reloadDStream(&seqState->DStream); + } + +# else if (mlBits > 0) seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); @@ -1335,6 +1347,8 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c if (llBits > 0) seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); +# endif + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); From 8964512f55047c5e98341024d63443eb2e3c41f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= Date: Sat, 3 Oct 2026 17:19:30 +0200 Subject: [PATCH 2/2] AArch64: Share decoder layout across Clang and GCC Compute FSE bit positions from a common starting position to avoid serial bitsConsumed updates and retain GCC throughput with the new length-read layout. Assisted-by: OpenAI Codex --- lib/decompress/zstd_decompress_block.c | 33 +++++++++++--------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index 3f30191ba27..a344a2bb25d 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -1316,7 +1316,6 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c seq.offset = offset; } -# if defined(__clang__) /* Most symbols need no additional literal-length bits. Keep their * match-length read separate from the uncommon two-length case. */ ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); @@ -1334,29 +1333,25 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c BIT_reloadDStream(&seqState->DStream); } -# else - if (mlBits > 0) - seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); - - if (UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - - /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ - ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - - if (llBits > 0) - seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); - -# endif - DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); if (!isLastSeq) { /* Don't update FSE state for last sequence. */ - ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + /* Compute each bit position from the same starting position to + * avoid serial updates of bitsConsumed. */ + { U32 const consumed = seqState->DStream.bitsConsumed; + U32 const llmlBits = llnbBits + mlnbBits; + U32 const allFseBits = llmlBits + ofnbBits; + BitContainerType const container = seqState->DStream.bitContainer; + seqState->stateLL.state = llNext + BIT_getMiddleBits( + container, 64 - (consumed + llnbBits), llnbBits); + seqState->stateML.state = mlNext + BIT_getMiddleBits( + container, 64 - (consumed + llmlBits), mlnbBits); + seqState->stateOffb.state = ofNext + BIT_getMiddleBits( + container, 64 - (consumed + allFseBits), ofnbBits); + seqState->DStream.bitsConsumed = consumed + allFseBits; + } BIT_reloadDStream(&seqState->DStream); } }