From 4023f9d0d7414cf1761352822b48ebbe53de796f Mon Sep 17 00:00:00 2001 From: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> Date: Sun, 4 Oct 2026 08:18:21 +0530 Subject: [PATCH] legacy/v0.4: size the streaming output buffer for one extra block The v0.4 streaming decoder sizes its output buffer at exactly the frame window. When a flushed block leaves less than a full block of room, it wraps the buffer back to offset zero, discarding the history that later sequences still reference through offsets within the window. Match sources then read bytes the current block has already overwritten, so the streaming API silently returns different output than the one-shot API for the same frame. The v0.5 decoder had the same flaw (issue #4553), and the v0.6 and v0.7 decoders already solve it by sizing the buffer at windowSize plus one block, which makes the wrap harmless. Apply the same sizing here. Add a crafted frame to the legacy test whose first block is smaller than the declared window and whose next block references it: before this change the streaming decode differs from the one-shot decode, after it they match. --- lib/legacy/zstd_v04.c | 5 +- tests/legacy.c | 152 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 1 deletion(-) diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 31c2052583d..47b5cfe554c 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -3424,7 +3424,10 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs case ZBUFFds_decodeHeader: /* apply header to create / resize buffers */ - { size_t const neededOutSize = (size_t)1 << zbc->params.windowLog; + /* note : also provide space for one full block, so that the + * output buffer never needs to wrap while a block decodes, + * keeping match sources valid (see v0.6 decoder). */ + { size_t const neededOutSize = ((size_t)1 << zbc->params.windowLog) + BLOCKSIZE; size_t const neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ if (zbc->inBuffSize < neededInSize) { free(zbc->inBuff); diff --git a/tests/legacy.c b/tests/legacy.c index 18a590f1d51..fb353f9e44a 100644 --- a/tests/legacy.c +++ b/tests/legacy.c @@ -64,6 +64,10 @@ const char* const FRAME_V07; /* content is at end of file, produced by v0.7.0 const char* const FRAME_V08; /* content is at end of file, produced by v0.8.0 */ const char* const BLOCK; /* content is at end of file */ +/* crafted frame, see definition at end of file */ +#define FRAME_V04_WRAP_SIZE 1137 +const char* const FRAME_V04_WRAP; + /* legacy formats older than v0.4 only offer a single-shot decoder : * ZSTD_decompressStream() answers version_unsupported on such frames @@ -228,6 +232,68 @@ static int testStreamingAPI(const char* compressed, size_t compressedSize, return error_code; } +/* testStreamingMatchesOneShot() : + * the one-shot decoder is the reference : the streaming decoder must + * produce the exact same bytes on the same frame. + * The output chunk is small and not block-aligned, so that flushes and + * internal output buffer wraps happen at unaligned offsets. */ +static int testStreamingMatchesOneShot(const char* frame, size_t frameSize, + size_t decodedSize, const char* name) +{ + /* the one-shot decoder needs some slack past the last byte for its + * in-place literal copies, hence the extra 64 bytes */ + char* const reference = (char*)malloc(decodedSize + 64); + char* const outBuff = (char*)malloc(decodedSize + 64); + ZSTD_DStream* const stream = ZSTD_createDStream(); + ZSTD_inBuffer input = { frame, frameSize, 0 }; + size_t outPos = 0; + int error_code = 0; + + if (reference == NULL || outBuff == NULL || stream == NULL) { + DISPLAY("ERROR: Not enough memory\n"); + error_code = 1; + } + + if (!error_code) { + size_t const ret = ZSTD_decompress(reference, decodedSize + 64, frame, frameSize); + if (ZSTD_isError(ret) || ret != decodedSize) { + DISPLAY("ERROR: %s: one-shot decoder failed: %s\n", + name, ZSTD_isError(ret) ? ZSTD_getErrorName(ret) : "wrong size"); + error_code = 1; + } + } + + while (!error_code) { + ZSTD_outBuffer output = { outBuff + outPos, 333, 0 }; + if ((size_t)(outPos + output.size) > decodedSize) output.size = decodedSize - outPos; + { size_t const ret = ZSTD_decompressStream(stream, &output, &input); + if (ZSTD_isError(ret)) { + DISPLAY("ERROR: %s: ZSTD_decompressStream: %s\n", name, ZSTD_getErrorName(ret)); + error_code = 1; + break; + } + outPos += output.pos; + if (ret == 0) break; + } + } + + if (!error_code && outPos != decodedSize) { + DISPLAY("ERROR: %s: streaming decoder produced %zu bytes instead of %zu\n", + name, outPos, decodedSize); + error_code = 1; + } + if (!error_code && memcmp(reference, outBuff, decodedSize) != 0) { + DISPLAY("ERROR: %s: streaming decoder output differs from one-shot decoder\n", name); + error_code = 1; + } + + free(reference); + free(outBuff); + ZSTD_freeDStream(stream); + if (error_code == 0) DISPLAY("%s : streaming matches one-shot\n", name); + return error_code; +} + static int testFrameDecoding(const char* compressed, size_t compressedSize, size_t expectedSize) { @@ -298,6 +364,13 @@ int main(void) * are what the streaming subset must decode to */ if (testStreamingAPI(streamable, streamableSize, expected, blockSize * nbStreamFrames)) break; +#if (ZSTD_LEGACY_SUPPORT <= 4) + /* crafted frame : its first block is smaller than the declared window, + * so the v0.4 streaming decoder wraps its output buffer while later + * sequences still reference data from that block */ + if (testStreamingMatchesOneShot(FRAME_V04_WRAP, FRAME_V04_WRAP_SIZE, + 1148, "v0.4 sub-window wrap frame")) break; +#endif result = 0; } while (0); @@ -426,3 +499,82 @@ const char* const BLOCK = "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"; + + +/* A crafted v0.4 frame (windowLog == 11, raw 600-byte first block, then a + * compressed block whose sequence offset reaches into the first block). + * It decodes to 1148 bytes; the streaming decoder must agree with the + * one-shot decoder. */ +const char* const FRAME_V04_WRAP = + "\x24\xB5\x2F\xFD\x00\x40\x02\x58\x30\x31\x32\x33\x34\x35\x36\x37" + "\x38\x39\x3A\x3B\x3C\x3D\x3E\x3F\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4A\x4B\x4C\x4D\x4E\x4F\x50\x51\x52\x53\x54\x55\x56\x57" + "\x58\x59\x5A\x5B\x5C\x5D\x5E\x5F\x60\x61\x62\x63\x64\x65\x66\x67" + "\x68\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75\x76\x77" + "\x78\x79\x7A\x7B\x7C\x7D\x7E\x7F\x80\x81\x82\x83\x84\x85\x86\x87" + "\x88\x89\x8A\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93\x94\x95\x96\x97" + "\x98\x99\x9A\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7" + "\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7" + "\xB8\xB9\xBA\xBB\xBC\xBD\xBE\xBF\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7" + "\xC8\xC9\xCA\xCB\xCC\xCD\xCE\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7" + "\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF\xE0\xE1\xE2\xE3\xE4\xE5\xE6\xE7" + "\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7" + "\xF8\xF9\xFA\xFB\xFC\xFD\xFE\xFF\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\x20\x21\x22\x23\x24\x25\x26\x27" + "\x28\x29\x2A\x2B\x2C\x2D\x2E\x2F\x30\x31\x32\x33\x34\x35\x36\x37" + "\x38\x39\x3A\x3B\x3C\x3D\x3E\x3F\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4A\x4B\x4C\x4D\x4E\x4F\x50\x51\x52\x53\x54\x55\x56\x57" + "\x58\x59\x5A\x5B\x5C\x5D\x5E\x5F\x60\x61\x62\x63\x64\x65\x66\x67" + "\x68\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75\x76\x77" + "\x78\x79\x7A\x7B\x7C\x7D\x7E\x7F\x80\x81\x82\x83\x84\x85\x86\x87" + "\x88\x89\x8A\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93\x94\x95\x96\x97" + "\x98\x99\x9A\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7" + "\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7" + "\xB8\xB9\xBA\xBB\xBC\xBD\xBE\xBF\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7" + "\xC8\xC9\xCA\xCB\xCC\xCD\xCE\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7" + "\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF\xE0\xE1\xE2\xE3\xE4\xE5\xE6\xE7" + "\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7" + "\xF8\xF9\xFA\xFB\xFC\xFD\xFE\xFF\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\x20\x21\x22\x23\x24\x25\x26\x27" + "\x28\x29\x2A\x2B\x2C\x2D\x2E\x2F\x30\x31\x32\x33\x34\x35\x36\x37" + "\x38\x39\x3A\x3B\x3C\x3D\x3E\x3F\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4A\x4B\x4C\x4D\x4E\x4F\x50\x51\x52\x53\x54\x55\x56\x57" + "\x58\x59\x5A\x5B\x5C\x5D\x5E\x5F\x60\x61\x62\x63\x64\x65\x66\x67" + "\x68\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75\x76\x77" + "\x78\x79\x7A\x7B\x7C\x7D\x7E\x7F\x80\x81\x82\x83\x84\x85\x86\x87" + "\x00\x02\x0B\xD1\x07\x00\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6A" + "\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7A" + "\x7B\x7C\x7D\x7E\x7F\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A" + "\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9A" + "\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA" + "\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA" + "\xBB\xBC\xBD\xBE\xBF\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA" + "\xCB\xCC\xCD\xCE\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA" + "\xDB\xDC\xDD\xDE\xDF\xE0\xE1\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA" + "\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7\xF8\xF9\xFA" + "\xFB\xFC\xFD\xFE\xFF\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A" + "\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A" + "\x1B\x1C\x1D\x1E\x1F\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2A" + "\x2B\x2C\x2D\x2E\x2F\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3A" + "\x3B\x3C\x3D\x3E\x3F\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4A" + "\x4B\x4C\x4D\x4E\x4F\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5A" + "\x5B\x5C\x5D\x5E\x5F\x60\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6A" + "\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7A" + "\x7B\x7C\x7D\x7E\x7F\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A" + "\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9A" + "\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA" + "\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA" + "\xBB\xBC\xBD\xBE\xBF\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA" + "\xCB\xCC\xCD\xCE\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA" + "\xDB\xDC\xDD\xDE\xDF\xE0\xE1\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA" + "\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7\xF8\xF9\xFA" + "\xFB\xFC\xFD\xFE\xFF\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A" + "\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A" + "\x1B\x1C\x1D\x1E\x1F\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2A" + "\x2B\x2C\x2D\x2E\x2F\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3A" + "\x3B\x3C\x3D\x3E\x3F\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4A" + "\x4B\x4C\x4D\x4E\x4F\x50\x51\x52\x53\x54\x03\x00\x54\x04\xFF\xF4" + "\x01\x00\x00\x00\x00\x00\x12\x00\x40\x0F\x00\x92\xFA\x03\xC0\x00" + "\x00";