From 4717a4fb3a07c7528ed5d004b134dddf8c17c45a Mon Sep 17 00:00:00 2001 From: tvost2 Date: Sun, 30 Aug 2026 23:34:52 -0300 Subject: [PATCH] opt: allow bitCost == scaleLog in ZSTD_rescaleFreqs (fixes #4681) When the optimal parser initializes its statistics from a dictionary's FSE tables, it reads a per-symbol cost with FSE_getMaxNbBits(). That function returns a "fake cost" of tableLog+1 for zero-frequency symbols. The litLength / matchLength FSE tables can use tableLog == 9 == scaleLog-1, so the fake cost is exactly scaleLog (10), which tripped assert(bitCost < scaleLog) in debug builds. The value computation right below already copes with it: 1 << (scaleLog - bitCost) == 1 << 0 == 1, i.e. identical to the release-build result and to the 'minimum to calculate cost' fallback. Relax the three FSE asserts to bitCost <= scaleLog, matching the literal loop just above which already uses HUF_getNbBitsFromCTable() the same way. Adds a fuzzer.c regression test that builds a CDict with an optimal-parser strategy from a 433-byte dictionary whose FSE NCount has zero-frequency matchlength symbols, then compresses and round-trips. --- lib/compress/zstd_opt.c | 11 ++++-- tests/fuzzer.c | 74 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 562a6b17bc9..c5437a43eac 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -180,7 +180,12 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, for (ll=0; ll<=MaxLL; ll++) { U32 const scaleLog = 10; /* scale to 1K */ U32 const bitCost = FSE_getMaxNbBits(llstate.symbolTT, ll); - assert(bitCost < scaleLog); + /* FSE_getMaxNbBits() returns a "fake cost" of tableLog+1 for + * zero-frequency symbols, and the FSE table log can be as + * large as scaleLog-1, so bitCost==scaleLog is reachable. + * The value computation below already handles it (1<<0==1), + * matching the literal-stats loop above. */ + assert(bitCost <= scaleLog); optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; optPtr->litLengthSum += optPtr->litLengthFreq[ll]; } } @@ -192,7 +197,7 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, for (ml=0; ml<=MaxML; ml++) { U32 const scaleLog = 10; U32 const bitCost = FSE_getMaxNbBits(mlstate.symbolTT, ml); - assert(bitCost < scaleLog); + assert(bitCost <= scaleLog); /* see litLength loop above */ optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; optPtr->matchLengthSum += optPtr->matchLengthFreq[ml]; } } @@ -204,7 +209,7 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, for (of=0; of<=MaxOff; of++) { U32 const scaleLog = 10; U32 const bitCost = FSE_getMaxNbBits(ofstate.symbolTT, of); - assert(bitCost < scaleLog); + assert(bitCost <= scaleLog); /* see litLength loop above */ optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; optPtr->offCodeSum += optPtr->offCodeFreq[of]; } } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 32360b9a58d..7a2c3de91e3 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -2080,6 +2080,80 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3d : optimal parser + dictionary with sparse FSE tables : ", testNb++); + { /* Regression test for #4681: a dictionary whose FSE NCount has + * zero-frequency symbols within 0..MaxML makes FSE_getMaxNbBits() + * return tableLog+1 == 10, which tripped assert(bitCost < scaleLog) + * in ZSTD_rescaleFreqs() for the litLength/matchLength/offCode loops. */ + static const BYTE dict[] = { + 0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f, + 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01, + 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18, 0x86, 0x61, + 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0x66, + 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61, 0x6c, 0x65, + 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65, 0x6c, 0x65, + 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69, 0x64, 0x69, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x20, 0x6d, 0x61, 0x74, + 0x63, 0x68, 0x20, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x64, + 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x20, 0x6d, 0x61, + 0x74, 0x63, 0x68, 0x20, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x20, 0x6d, + 0x61, 0x74, 0x63, 0x68, 0x20, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x61, 0x64, 0x69, 0x63, 0x74, 0x61, 0x72, 0x79, 0x20, 0x6d, 0x61, 0x74, + 0x63, 0x68, 0x20, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x64, + 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x20, 0x6d, 0x61, + 0x74, 0x63, 0x68, 0x20, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18, 0x86, + 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x72, 0x79, + 0x20, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x20, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x61, 0x79, 0x20, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x20, 0x61, 0x61, 0x61, + 0x61, 0x61, 0x61, 0x61, 0x61, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x61, 0x72, 0x79, 0x20, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x20, 0x61, 0x61, + 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x61, 0x72, 0x79, 0x20, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x20, 0x61, + 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x64, 0x69, 0x63, 0x74, 0x69, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x30, 0x31, 0x32, 0x33, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x10, 0x00, 0x02, + 0x6f + }; + static const char* const src = + "0123453456789abcdef0123 dictionary match aaaaaaaa\n"; + size_t const srcSize = strlen(src); + int strategy; + for (strategy = ZSTD_btopt; strategy <= ZSTD_btultra2; strategy++) { + ZSTD_compressionParameters cParams = ZSTD_getCParams(1, srcSize, sizeof(dict)); + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + ZSTD_CDict* cdict; + size_t cSize; + cParams.strategy = (ZSTD_strategy)strategy; + cParams = ZSTD_adjustCParams(cParams, srcSize, sizeof(dict)); + cdict = ZSTD_createCDict_advanced(dict, sizeof(dict), ZSTD_dlm_byRef, + ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); + CHECK(cdict != NULL); + cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, + src, srcSize, cdict); + CHECK_Z(cSize); /* must not assert / error */ + { size_t const dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, + compressedBuffer, cSize, dict, sizeof(dict)); + CHECK_Z(dSize); + CHECK(dSize == srcSize && !memcmp(decodedBuffer, src, srcSize)); + } + ZSTD_freeCDict(cdict); + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + } + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_outBuffer out = {NULL, 0, 0};