From b664c62e2061cca4296cedf27e5d8d8bf2337287 Mon Sep 17 00:00:00 2001 From: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> Date: Sun, 4 Oct 2026 10:05:16 +0530 Subject: [PATCH] legacy/v0.7: decode RLE blocks in the streaming path The v0.7 decoder supports RLE blocks in its one-shot path, expanding them with ZSTDv07_generateNxBytes(), but the block stage of ZSTDv07_decompressContinue() rejects them with a generic error. A frame containing an RLE block therefore decodes through ZSTD_decompress() and fails through ZSTD_decompressStream(), partway through the frame. Transfer the block's regenerated size from the header stage, where ZSTDv07_getcBlockSize() already extracts it into blockProperties, to the block stage, and expand RLE blocks there with the same helper the one-shot path uses. The v0.7.0 reference decoder shipped with the same inconsistency, so no frame produced by the v0.7 line itself contains an RLE block; the asymmetry is nonetheless reachable by any third party encoder for the format, and the one-shot path defines what the format accepts. Add a crafted frame to the legacy test, one RLE block expanding to 100000 bytes: before this change the streaming leg fails with Error (generic), after it the streaming output matches the one-shot output. --- lib/legacy/zstd_v07.c | 4 ++- tests/legacy.c | 77 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index cdc56288cb2..cd3c69b412c 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -2885,6 +2885,7 @@ struct ZSTDv07_DCtx_s U32 rep[3]; ZSTDv07_frameParams fParams; blockType_t bType; /* used in ZSTDv07_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ + U32 rleSize; /* for bt_rle blocks : regenerated size, also transferred between the two stages */ ZSTDv07_dStage stage; U32 litEntropy; U32 fseEntropy; @@ -3982,6 +3983,7 @@ size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapac } else { dctx->expected = cBlockSize; dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; /* only meaningful for bt_rle */ dctx->stage = ZSTDds_decompressBlock; } return 0; @@ -3997,7 +3999,7 @@ size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapac rSize = ZSTDv07_copyRawBlock(dst, dstCapacity, src, srcSize); break; case bt_rle : - return ERROR(GENERIC); /* not yet handled */ + rSize = ZSTDv07_generateNxBytes(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); break; case bt_end : /* should never happen (filtered at phase 1) */ rSize = 0; diff --git a/tests/legacy.c b/tests/legacy.c index 18a590f1d51..3eb2b5d0e67 100644 --- a/tests/legacy.c +++ b/tests/legacy.c @@ -64,6 +64,10 @@ const char* const FRAME_V07; /* content is at end of file, produced by v0.7.0 const char* const FRAME_V08; /* content is at end of file, produced by v0.8.0 */ const char* const BLOCK; /* content is at end of file */ +/* crafted frame, see definition at end of file */ +#define FRAME_V07_RLE_SIZE 16 +const char* const FRAME_V07_RLE; + /* legacy formats older than v0.4 only offer a single-shot decoder : * ZSTD_decompressStream() answers version_unsupported on such frames @@ -228,6 +232,66 @@ static int testStreamingAPI(const char* compressed, size_t compressedSize, return error_code; } +/* testStreamingMatchesOneShot() : + * the one-shot decoder is the reference : the streaming decoder must + * produce the exact same bytes on the same frame. */ +static int testStreamingMatchesOneShot(const char* frame, size_t frameSize, + size_t decodedSize, const char* name) +{ + /* the one-shot decoder needs some slack past the last byte for its + * in-place literal copies, hence the extra 64 bytes */ + char* const reference = (char*)malloc(decodedSize + 64); + char* const outBuff = (char*)malloc(decodedSize + 64); + ZSTD_DStream* const stream = ZSTD_createDStream(); + ZSTD_inBuffer input = { frame, frameSize, 0 }; + size_t outPos = 0; + int error_code = 0; + + if (reference == NULL || outBuff == NULL || stream == NULL) { + DISPLAY("ERROR: Not enough memory\n"); + error_code = 1; + } + + if (!error_code) { + size_t const ret = ZSTD_decompress(reference, decodedSize + 64, frame, frameSize); + if (ZSTD_isError(ret) || ret != decodedSize) { + DISPLAY("ERROR: %s: one-shot decoder failed: %s\n", + name, ZSTD_isError(ret) ? ZSTD_getErrorName(ret) : "wrong size"); + error_code = 1; + } + } + + while (!error_code) { + ZSTD_outBuffer output = { outBuff + outPos, 333, 0 }; + if ((size_t)(outPos + output.size) > decodedSize) output.size = decodedSize - outPos; + { size_t const ret = ZSTD_decompressStream(stream, &output, &input); + if (ZSTD_isError(ret)) { + DISPLAY("ERROR: %s: ZSTD_decompressStream: %s\n", name, ZSTD_getErrorName(ret)); + error_code = 1; + break; + } + outPos += output.pos; + if (ret == 0) break; + } + } + + if (!error_code && outPos != decodedSize) { + DISPLAY("ERROR: %s: streaming decoder produced %zu bytes instead of %zu\n", + name, outPos, decodedSize); + error_code = 1; + } + if (!error_code && memcmp(reference, outBuff, decodedSize) != 0) { + DISPLAY("ERROR: %s: streaming decoder output differs from one-shot decoder\n", name); + error_code = 1; + } + + free(reference); + free(outBuff); + ZSTD_freeDStream(stream); + if (error_code == 0) DISPLAY("%s : streaming matches one-shot\n", name); + return error_code; +} + static int testFrameDecoding(const char* compressed, size_t compressedSize, size_t expectedSize) { @@ -298,6 +362,12 @@ int main(void) * are what the streaming subset must decode to */ if (testStreamingAPI(streamable, streamableSize, expected, blockSize * nbStreamFrames)) break; +#if (ZSTD_LEGACY_SUPPORT <= 7) + /* crafted frame with a single RLE block : the one-shot v0.7 decoder + * expands it, and the streaming one must agree */ + if (testStreamingMatchesOneShot(FRAME_V07_RLE, FRAME_V07_RLE_SIZE, + 100000, "v0.7 RLE block frame")) break; +#endif result = 0; } while (0); @@ -426,3 +496,10 @@ const char* const BLOCK = "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"; + + +/* A crafted v0.7 frame (magic, direct mode header with a 4 byte frame content + * size of 100000, one RLE block of byte 0x41, end block). The one-shot decoder + * expands RLE blocks; the streaming decoder must agree. */ +const char* const FRAME_V07_RLE = + "\x27\xB5\x2F\xFD\xA0\xA0\x86\x01\x00\x81\x86\xA0\x41\xC0\x00\x00";