diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 31c2052583d..47b5cfe554c 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -3424,7 +3424,10 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs case ZBUFFds_decodeHeader: /* apply header to create / resize buffers */ - { size_t const neededOutSize = (size_t)1 << zbc->params.windowLog; + /* note : also provide space for one full block, so that the + * output buffer never needs to wrap while a block decodes, + * keeping match sources valid (see v0.6 decoder). */ + { size_t const neededOutSize = ((size_t)1 << zbc->params.windowLog) + BLOCKSIZE; size_t const neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ if (zbc->inBuffSize < neededInSize) { free(zbc->inBuff); diff --git a/tests/legacy.c b/tests/legacy.c index 18a590f1d51..fb353f9e44a 100644 --- a/tests/legacy.c +++ b/tests/legacy.c @@ -64,6 +64,10 @@ const char* const FRAME_V07; /* content is at end of file, produced by v0.7.0 const char* const FRAME_V08; /* content is at end of file, produced by v0.8.0 */ const char* const BLOCK; /* content is at end of file */ +/* crafted frame, see definition at end of file */ +#define FRAME_V04_WRAP_SIZE 1137 +const char* const FRAME_V04_WRAP; + /* legacy formats older than v0.4 only offer a single-shot decoder : * ZSTD_decompressStream() answers version_unsupported on such frames @@ -228,6 +232,68 @@ static int testStreamingAPI(const char* compressed, size_t compressedSize, return error_code; } +/* testStreamingMatchesOneShot() : + * the one-shot decoder is the reference : the streaming decoder must + * produce the exact same bytes on the same frame. + * The output chunk is small and not block-aligned, so that flushes and + * internal output buffer wraps happen at unaligned offsets. */ +static int testStreamingMatchesOneShot(const char* frame, size_t frameSize, + size_t decodedSize, const char* name) +{ + /* the one-shot decoder needs some slack past the last byte for its + * in-place literal copies, hence the extra 64 bytes */ + char* const reference = (char*)malloc(decodedSize + 64); + char* const outBuff = (char*)malloc(decodedSize + 64); + ZSTD_DStream* const stream = ZSTD_createDStream(); + ZSTD_inBuffer input = { frame, frameSize, 0 }; + size_t outPos = 0; + int error_code = 0; + + if (reference == NULL || outBuff == NULL || stream == NULL) { + DISPLAY("ERROR: Not enough memory\n"); + error_code = 1; + } + + if (!error_code) { + size_t const ret = ZSTD_decompress(reference, decodedSize + 64, frame, frameSize); + if (ZSTD_isError(ret) || ret != decodedSize) { + DISPLAY("ERROR: %s: one-shot decoder failed: %s\n", + name, ZSTD_isError(ret) ? ZSTD_getErrorName(ret) : "wrong size"); + error_code = 1; + } + } + + while (!error_code) { + ZSTD_outBuffer output = { outBuff + outPos, 333, 0 }; + if ((size_t)(outPos + output.size) > decodedSize) output.size = decodedSize - outPos; + { size_t const ret = ZSTD_decompressStream(stream, &output, &input); + if (ZSTD_isError(ret)) { + DISPLAY("ERROR: %s: ZSTD_decompressStream: %s\n", name, ZSTD_getErrorName(ret)); + error_code = 1; + break; + } + outPos += output.pos; + if (ret == 0) break; + } + } + + if (!error_code && outPos != decodedSize) { + DISPLAY("ERROR: %s: streaming decoder produced %zu bytes instead of %zu\n", + name, outPos, decodedSize); + error_code = 1; + } + if (!error_code && memcmp(reference, outBuff, decodedSize) != 0) { + DISPLAY("ERROR: %s: streaming decoder output differs from one-shot decoder\n", name); + error_code = 1; + } + + free(reference); + free(outBuff); + ZSTD_freeDStream(stream); + if (error_code == 0) DISPLAY("%s : streaming matches one-shot\n", name); + return error_code; +} + static int testFrameDecoding(const char* compressed, size_t compressedSize, size_t expectedSize) { @@ -298,6 +364,13 @@ int main(void) * are what the streaming subset must decode to */ if (testStreamingAPI(streamable, streamableSize, expected, blockSize * nbStreamFrames)) break; +#if (ZSTD_LEGACY_SUPPORT <= 4) + /* crafted frame : its first block is smaller than the declared window, + * so the v0.4 streaming decoder wraps its output buffer while later + * sequences still reference data from that block */ + if (testStreamingMatchesOneShot(FRAME_V04_WRAP, FRAME_V04_WRAP_SIZE, + 1148, "v0.4 sub-window wrap frame")) break; +#endif result = 0; } while (0); @@ -426,3 +499,82 @@ const char* const BLOCK = "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"; + + +/* A crafted v0.4 frame (windowLog == 11, raw 600-byte first block, then a + * compressed block whose sequence offset reaches into the first block). + * It decodes to 1148 bytes; the streaming decoder must agree with the + * one-shot decoder. */ +const char* const FRAME_V04_WRAP = + "\x24\xB5\x2F\xFD\x00\x40\x02\x58\x30\x31\x32\x33\x34\x35\x36\x37" + "\x38\x39\x3A\x3B\x3C\x3D\x3E\x3F\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4A\x4B\x4C\x4D\x4E\x4F\x50\x51\x52\x53\x54\x55\x56\x57" + "\x58\x59\x5A\x5B\x5C\x5D\x5E\x5F\x60\x61\x62\x63\x64\x65\x66\x67" + "\x68\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75\x76\x77" + "\x78\x79\x7A\x7B\x7C\x7D\x7E\x7F\x80\x81\x82\x83\x84\x85\x86\x87" + "\x88\x89\x8A\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93\x94\x95\x96\x97" + "\x98\x99\x9A\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7" + "\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7" + "\xB8\xB9\xBA\xBB\xBC\xBD\xBE\xBF\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7" + "\xC8\xC9\xCA\xCB\xCC\xCD\xCE\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7" + "\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF\xE0\xE1\xE2\xE3\xE4\xE5\xE6\xE7" + "\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7" + "\xF8\xF9\xFA\xFB\xFC\xFD\xFE\xFF\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\x20\x21\x22\x23\x24\x25\x26\x27" + "\x28\x29\x2A\x2B\x2C\x2D\x2E\x2F\x30\x31\x32\x33\x34\x35\x36\x37" + "\x38\x39\x3A\x3B\x3C\x3D\x3E\x3F\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4A\x4B\x4C\x4D\x4E\x4F\x50\x51\x52\x53\x54\x55\x56\x57" + "\x58\x59\x5A\x5B\x5C\x5D\x5E\x5F\x60\x61\x62\x63\x64\x65\x66\x67" + "\x68\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75\x76\x77" + "\x78\x79\x7A\x7B\x7C\x7D\x7E\x7F\x80\x81\x82\x83\x84\x85\x86\x87" + "\x88\x89\x8A\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93\x94\x95\x96\x97" + "\x98\x99\x9A\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7" + "\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7" + "\xB8\xB9\xBA\xBB\xBC\xBD\xBE\xBF\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7" + "\xC8\xC9\xCA\xCB\xCC\xCD\xCE\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7" + "\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF\xE0\xE1\xE2\xE3\xE4\xE5\xE6\xE7" + "\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7" + "\xF8\xF9\xFA\xFB\xFC\xFD\xFE\xFF\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\x20\x21\x22\x23\x24\x25\x26\x27" + "\x28\x29\x2A\x2B\x2C\x2D\x2E\x2F\x30\x31\x32\x33\x34\x35\x36\x37" + "\x38\x39\x3A\x3B\x3C\x3D\x3E\x3F\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4A\x4B\x4C\x4D\x4E\x4F\x50\x51\x52\x53\x54\x55\x56\x57" + "\x58\x59\x5A\x5B\x5C\x5D\x5E\x5F\x60\x61\x62\x63\x64\x65\x66\x67" + "\x68\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75\x76\x77" + "\x78\x79\x7A\x7B\x7C\x7D\x7E\x7F\x80\x81\x82\x83\x84\x85\x86\x87" + "\x00\x02\x0B\xD1\x07\x00\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6A" + "\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7A" + "\x7B\x7C\x7D\x7E\x7F\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A" + "\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9A" + "\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA" + "\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA" + "\xBB\xBC\xBD\xBE\xBF\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA" + "\xCB\xCC\xCD\xCE\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA" + "\xDB\xDC\xDD\xDE\xDF\xE0\xE1\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA" + "\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7\xF8\xF9\xFA" + "\xFB\xFC\xFD\xFE\xFF\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A" + "\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A" + "\x1B\x1C\x1D\x1E\x1F\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2A" + "\x2B\x2C\x2D\x2E\x2F\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3A" + "\x3B\x3C\x3D\x3E\x3F\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4A" + "\x4B\x4C\x4D\x4E\x4F\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5A" + "\x5B\x5C\x5D\x5E\x5F\x60\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6A" + "\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7A" + "\x7B\x7C\x7D\x7E\x7F\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A" + "\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9A" + "\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA" + "\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA" + "\xBB\xBC\xBD\xBE\xBF\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA" + "\xCB\xCC\xCD\xCE\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA" + "\xDB\xDC\xDD\xDE\xDF\xE0\xE1\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA" + "\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7\xF8\xF9\xFA" + "\xFB\xFC\xFD\xFE\xFF\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A" + "\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A" + "\x1B\x1C\x1D\x1E\x1F\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2A" + "\x2B\x2C\x2D\x2E\x2F\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3A" + "\x3B\x3C\x3D\x3E\x3F\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4A" + "\x4B\x4C\x4D\x4E\x4F\x50\x51\x52\x53\x54\x03\x00\x54\x04\xFF\xF4" + "\x01\x00\x00\x00\x00\x00\x12\x00\x40\x0F\x00\x92\xFA\x03\xC0\x00" + "\x00";