diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 325f0b54dfc..99b201ad1ba 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -810,6 +810,40 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) return ZSTD_findFrameCompressedSize_advanced(src, srcSize, ZSTD_f_zstd1); } +/** ZSTD_findFrameBoundaries() : + * See docs in zstd.h + * Note: compatible with legacy mode */ +size_t ZSTD_findFrameBoundaries(const void* src, size_t srcSize, + ZSTD_FrameBoundary* frameBoundaries, + size_t frameBoundariesCapacity) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + size_t remainingSize = srcSize; + size_t nbFrames = 0; + + while (remainingSize > 0) { + size_t const frameSize = ZSTD_findFrameCompressedSize(ip, remainingSize); + if (ZSTD_isError(frameSize)) return frameSize; + /* ZSTD_findFrameCompressedSize() only succeeds when the full frame is + * present, hence frameSize <= remainingSize and frameSize > 0. */ + assert(frameSize <= remainingSize); + assert(frameSize > 0); + if (frameBoundaries != NULL) { + if (nbFrames >= frameBoundariesCapacity) + return ZSTD_ERROR(dstSize_tooSmall); + frameBoundaries[nbFrames].offset = (size_t)(ip - istart); + frameBoundaries[nbFrames].compressedSize = frameSize; + frameBoundaries[nbFrames].isSkippable = + ((MEM_readLE32(ip) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START); + } + nbFrames++; + ip += frameSize; + remainingSize -= frameSize; + } + return nbFrames; +} + /** ZSTD_decompressBound() : * compatible with legacy mode * `src` must point to the start of a ZSTD frame or a skippable frame diff --git a/lib/zstd.h b/lib/zstd.h index 97fef316fd6..7ed189d7eab 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -226,6 +226,44 @@ ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t * which is always equal to its content size + 8 bytes for headers. */ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); +/*! ZSTD_FrameBoundary : describes the location of a single frame + * within a scanned buffer, as reported by ZSTD_findFrameBoundaries(). */ +typedef struct { + size_t offset; /* offset of the frame's first byte (magic number) from the start of the scanned buffer */ + size_t compressedSize; /* compressed size of the frame in bytes, including headers and checksum */ + unsigned isSkippable; /* 1 if the frame is a skippable frame, 0 for a standard (or legacy) zstd frame */ +} ZSTD_FrameBoundary; + +/*! ZSTD_findFrameBoundaries() : + * Scans `src` (`srcSize` bytes) and locates every zstd frame it contains, + * without decompressing any content. + * Each frame's magic number is validated, and frame extents are determined + * exactly as in ZSTD_findFrameCompressedSize(): standard frames are walked + * block by block, skippable frames use their embedded size field. + * This is the enabling primitive for frame-granularity parallel decompression: + * each reported frame can be handed to ZSTD_decompress() independently. + * + * On success, fills up to `frameBoundariesCapacity` entries of `frameBoundaries[]`, + * in buffer order, so that `frameBoundaries[i].offset + frameBoundaries[i].compressedSize + * == frameBoundaries[i+1].offset`, and the last frame ends exactly at `srcSize`. + * + * If `frameBoundaries == NULL`, no entries are written: the input is still fully + * validated and the function simply returns the number of frames it contains. + * This provides the two-pass idiom: call once with NULL to obtain the count, + * allocate, then call again to fill the array. + * + * @return : the number of frames found, + * or an error code (test with ZSTD_isError()) if the input is invalid: + * - ZSTD_ERROR(prefix_unknown) : a frame starts with an invalid magic number + * - ZSTD_ERROR(srcSize_wrong) : the input ends in the middle of a frame + * - ZSTD_ERROR(dstSize_tooSmall): more frames than `frameBoundariesCapacity` + * Note: an empty input (`srcSize == 0`) is valid and yields 0 frames. + * Note: legacy frames are reported like standard frames when the library + * was built with legacy support; otherwise their magic is rejected. */ +ZSTDLIB_API size_t ZSTD_findFrameBoundaries(const void* src, size_t srcSize, + ZSTD_FrameBoundary* frameBoundaries, + size_t frameBoundariesCapacity); + /*====== Compression helper functions ======*/ diff --git a/tests/Makefile b/tests/Makefile index e268ddeb9e4..b8b45816015 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -236,6 +236,9 @@ largeDictionary: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c largeDictionary.c CLEAN += invalidDictionaries invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c +CLEAN += frameScannerTest +frameScannerTest : $(ZSTD_OBJECTS) frameScannerTest.c + CLEAN += legacy # legacy.c adapts to the level it is compiled with : it only tests the frames # this level can decode. Default to the oldest format, to cover all decoders. @@ -330,7 +333,7 @@ check: test-zstd fuzztest: test-fuzzer test-zstream test-decodecorpus .PHONY: test -test: test-zstd test-cli-tests test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus +test: test-zstd test-cli-tests test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-frameScanner test-legacy test-decodecorpus ifeq ($(QEMU_SYS),) test: test-pool endif @@ -412,6 +415,9 @@ test-largeDictionary: largeDictionary test-invalidDictionaries: invalidDictionaries $(QEMU_SYS) ./invalidDictionaries +test-frameScanner: frameScannerTest + $(QEMU_SYS) ./frameScannerTest + test-legacy: legacy $(QEMU_SYS) ./legacy diff --git a/tests/frameScannerTest.c b/tests/frameScannerTest.c new file mode 100644 index 00000000000..e988e312ce5 --- /dev/null +++ b/tests/frameScannerTest.c @@ -0,0 +1,247 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Unit tests for ZSTD_findFrameBoundaries() (issue #4734). + * Builds a multi-frame buffer (standard + skippable frames), verifies that + * the scanner reports exact, contiguous boundaries, and proves the + * concurrency use case: every reported standard frame decompresses + * independently to its original content. */ + +#define ZSTD_STATIC_LINKING_ONLY /* for ZSTD_writeSkippableFrame() */ +#include "zstd.h" + +#include +#include +#include + +static int g_failures = 0; +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + printf("FAIL %s:%d: ", __FILE__, __LINE__); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + g_failures++; \ + } \ + } while (0) + +/* simple xorshift64 PRNG */ +static unsigned long long g_rng = 0x123456789ABCDEFULL; +static unsigned rngNext(void) { + g_rng ^= g_rng << 13; + g_rng ^= g_rng >> 7; + g_rng ^= g_rng << 17; + return (unsigned)(g_rng >> 11); +} + +/* Fill `dst` with compressible pseudo-random data (low entropy). */ +static void genCompressible(void* dst, size_t size, unsigned seed) { + unsigned char* p = (unsigned char*)dst; + size_t i; + g_rng = 0x123456789ABCDEFULL + seed; + for (i = 0; i < size; i++) { + /* mostly small alphabet => compresses well, with occasional runs */ + unsigned r = rngNext(); + p[i] = (unsigned char)((r % 7) + ((r >> 8) % 3) * 40 + (i % 251 == 0 ? 200 : 0)); + } +} + +#define NB_STD_FRAMES 5 + +int main(void) { + static const size_t stdSizes[NB_STD_FRAMES] = { 1024, 65536, 13, 300000, 1 }; + static const int stdLevels[NB_STD_FRAMES] = { 1, 3, 19, 5, 1 }; + void* stdSrc[NB_STD_FRAMES]; + size_t stdFrameCap = 0; + unsigned char* multiBuf = NULL; + size_t multiSize = 0, multiCap = 0; + size_t i; + + /* ---- build source blobs ---- */ + for (i = 0; i < NB_STD_FRAMES; i++) { + stdSrc[i] = malloc(stdSizes[i] ? stdSizes[i] : 1); + CHECK(stdSrc[i] != NULL, "malloc failed"); + genCompressible(stdSrc[i], stdSizes[i], (unsigned)(100 + i)); + stdFrameCap += ZSTD_compressBound(stdSizes[i]); + } + /* room for frames + 2 skippable frames + slack */ + multiCap = stdFrameCap + 2 * (8 + 4096) + 1024; + multiBuf = (unsigned char*)malloc(multiCap); + CHECK(multiBuf != NULL, "malloc failed"); + + /* ---- frame 0,1 : standard frames ---- */ + for (i = 0; i < 2; i++) { + size_t cSize = ZSTD_compress(multiBuf + multiSize, multiCap - multiSize, + stdSrc[i], stdSizes[i], stdLevels[i]); + CHECK(!ZSTD_isError(cSize), "compress failed: %s", ZSTD_getErrorName(cSize)); + multiSize += cSize; + } + /* ---- skippable frame A (magic variant 0) ---- */ + { + const char skipContent[] = "skippable-payload-A"; + size_t wSize = ZSTD_writeSkippableFrame(multiBuf + multiSize, multiCap - multiSize, + skipContent, sizeof(skipContent), 0); + CHECK(!ZSTD_isError(wSize), "writeSkippableFrame failed: %s", ZSTD_getErrorName(wSize)); + multiSize += wSize; + } + /* ---- frames 2,3,4 : standard frames ---- */ + for (i = 2; i < NB_STD_FRAMES; i++) { + size_t cSize = ZSTD_compress(multiBuf + multiSize, multiCap - multiSize, + stdSrc[i], stdSizes[i], stdLevels[i]); + CHECK(!ZSTD_isError(cSize), "compress failed: %s", ZSTD_getErrorName(cSize)); + multiSize += cSize; + } + /* ---- skippable frame B (magic variant 15, empty content) ---- */ + { + size_t wSize = ZSTD_writeSkippableFrame(multiBuf + multiSize, multiCap - multiSize, + NULL, 0, 15); + CHECK(!ZSTD_isError(wSize), "writeSkippableFrame failed: %s", ZSTD_getErrorName(wSize)); + multiSize += wSize; + } + + /* total frames: 5 standard + 2 skippable = 7 */ + printf("test buffer: %zu bytes, 7 frames (5 standard + 2 skippable)\n", multiSize); + + /* ---- counting pass (two-pass idiom) ---- */ + { + size_t const n = ZSTD_findFrameBoundaries(multiBuf, multiSize, NULL, 0); + CHECK(!ZSTD_isError(n), "counting pass failed: %s", ZSTD_getErrorName(n)); + CHECK(n == 7, "counting pass: expected 7 frames, got %zu", n); + } + + /* ---- fill pass ---- */ + { + ZSTD_FrameBoundary bounds[8]; + size_t const n = ZSTD_findFrameBoundaries(multiBuf, multiSize, bounds, 8); + size_t pos = 0, stdIdx = 0; + CHECK(!ZSTD_isError(n), "fill pass failed: %s", ZSTD_getErrorName(n)); + CHECK(n == 7, "fill pass: expected 7 frames, got %zu", n); + for (i = 0; i < n; i++) { + /* contiguity */ + CHECK(bounds[i].offset == pos, + "frame %zu: offset %zu, expected %zu", i, bounds[i].offset, pos); + /* size matches the single-frame primitive */ + { + size_t const single = ZSTD_findFrameCompressedSize(multiBuf + pos, multiSize - pos); + CHECK(!ZSTD_isError(single), "findFrameCompressedSize failed at frame %zu", i); + CHECK(bounds[i].compressedSize == single, + "frame %zu: size %zu != primitive %zu", i, bounds[i].compressedSize, single); + } + pos += bounds[i].compressedSize; + } + CHECK(pos == multiSize, "frames cover %zu of %zu bytes", pos, multiSize); + + /* skippable flags: frames 2 and 6 are the skippable ones */ + for (i = 0; i < n; i++) { + int expectSkip = (i == 2 || i == 6); + CHECK((bounds[i].isSkippable != 0) == expectSkip, + "frame %zu: isSkippable=%u, expected %d", i, bounds[i].isSkippable, expectSkip); + } + + /* concurrency proof: each standard frame decompresses independently */ + for (i = 0; i < n; i++) { + if (bounds[i].isSkippable) continue; + { + void* dec = malloc(stdSizes[stdIdx] ? stdSizes[stdIdx] : 1); + size_t const dSize = ZSTD_decompress(dec, stdSizes[stdIdx], + multiBuf + bounds[i].offset, + bounds[i].compressedSize); + CHECK(!ZSTD_isError(dSize), "decompress of frame %zu failed: %s", + i, ZSTD_getErrorName(dSize)); + CHECK(dSize == stdSizes[stdIdx], + "frame %zu: decompressed %zu bytes, expected %zu", + i, dSize, stdSizes[stdIdx]); + CHECK(memcmp(dec, stdSrc[stdIdx], stdSizes[stdIdx]) == 0, + "frame %zu: roundtrip mismatch", i); + free(dec); + stdIdx++; + } + } + CHECK(stdIdx == NB_STD_FRAMES, "decompressed %zu standard frames, expected %d", + stdIdx, NB_STD_FRAMES); + } + + /* ---- capacity too small ---- */ + { + ZSTD_FrameBoundary bounds[2]; + size_t const r = ZSTD_findFrameBoundaries(multiBuf, multiSize, bounds, 2); + CHECK(ZSTD_isError(r), "small capacity: expected error, got %zu", r); + CHECK(ZSTD_getErrorCode(r) == ZSTD_error_dstSize_tooSmall, + "small capacity: expected dstSize_tooSmall, got %s", ZSTD_getErrorName(r)); + } + + /* ---- empty input ---- */ + { + size_t const r = ZSTD_findFrameBoundaries(multiBuf, 0, NULL, 0); + CHECK(!ZSTD_isError(r) && r == 0, "empty input: expected 0, got %s/%zu", + ZSTD_isError(r) ? ZSTD_getErrorName(r) : "ok", r); + } + + /* ---- truncated inputs ---- */ + { + static const size_t cutoffs[] = { 1, 3, 7, 100 }; + for (i = 0; i < sizeof(cutoffs) / sizeof(cutoffs[0]); i++) { + size_t cut = multiSize - cutoffs[i]; + size_t const r = ZSTD_findFrameBoundaries(multiBuf, cut, NULL, 0); + CHECK(ZSTD_isError(r), "truncated at %zu: expected error, got %zu", cut, r); + } + /* cut exactly at a frame boundary must succeed with fewer frames */ + { + ZSTD_FrameBoundary b0[1]; + size_t const f0 = ZSTD_findFrameCompressedSize(multiBuf, multiSize); + size_t const r = ZSTD_findFrameBoundaries(multiBuf, f0, b0, 1); + CHECK(!ZSTD_isError(r) && r == 1, + "boundary cut: expected 1 frame, got %s/%zu", + ZSTD_isError(r) ? ZSTD_getErrorName(r) : "ok", r); + CHECK(b0[0].offset == 0 && b0[0].compressedSize == f0, + "boundary cut: wrong boundary values"); + } + } + + /* ---- bad magic ---- */ + { + unsigned char* bad = (unsigned char*)malloc(multiSize); + size_t r; + memcpy(bad, multiBuf, multiSize); + bad[0] ^= 0xFF; bad[1] ^= 0xFF; bad[2] ^= 0xFF; bad[3] ^= 0xFF; + r = ZSTD_findFrameBoundaries(bad, multiSize, NULL, 0); + CHECK(ZSTD_isError(r), "bad magic: expected error, got %zu", r); + free(bad); + } + /* pure garbage */ + { + static const unsigned char garbage[64] = { + 0xDE, 0xAD, 0xBE, 0xEF, 1, 2, 3, 4, 5, 6, 7, 8 + }; + size_t const r = ZSTD_findFrameBoundaries(garbage, sizeof(garbage), NULL, 0); + CHECK(ZSTD_isError(r), "garbage: expected error, got %zu", r); + } + + /* ---- single skippable frame only ---- */ + { + unsigned char skipBuf[64]; + ZSTD_FrameBoundary b[1]; + size_t wSize = ZSTD_writeSkippableFrame(skipBuf, sizeof(skipBuf), "abc", 3, 7); + size_t r; + CHECK(!ZSTD_isError(wSize), "writeSkippableFrame failed"); + r = ZSTD_findFrameBoundaries(skipBuf, wSize, b, 1); + CHECK(!ZSTD_isError(r) && r == 1, "single skippable: expected 1, got %s/%zu", + ZSTD_isError(r) ? ZSTD_getErrorName(r) : "ok", r); + CHECK(b[0].offset == 0 && b[0].compressedSize == wSize && b[0].isSkippable, + "single skippable: wrong boundary values"); + } + + for (i = 0; i < NB_STD_FRAMES; i++) free(stdSrc[i]); + free(multiBuf); + + if (g_failures == 0) printf("frameScannerTest: all tests passed\n"); + else printf("frameScannerTest: %d FAILURES\n", g_failures); + return g_failures != 0; +} diff --git a/tests/fuzz/zstd_frame_info.c b/tests/fuzz/zstd_frame_info.c index 4d17bba7410..0cadaa01ce5 100644 --- a/tests/fuzz/zstd_frame_info.c +++ b/tests/fuzz/zstd_frame_info.c @@ -39,5 +39,41 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) ZSTD_isFrame(src, size); ZSTD_getFrameHeader(&zfh, src, size); ZSTD_getFrameHeader_advanced(&zfh, src, size, ZSTD_f_zstd1); + /* Frame-boundary scanner (issue #4734): validate structural invariants. */ + { + /* 1024 entries is plenty: smallest possible frame is 6 bytes, and we + * accept dstSize_tooSmall as a valid outcome for denser inputs. */ + ZSTD_FrameBoundary bounds[1024]; + size_t const counted = ZSTD_findFrameBoundaries(src, size, NULL, 0); + size_t const filled = ZSTD_findFrameBoundaries(src, size, bounds, 1024); + if (!ZSTD_isError(counted) && !ZSTD_isError(filled)) { + size_t i; + FUZZ_ASSERT(counted == filled); + FUZZ_ASSERT(filled <= 1024); + for (i = 0; i < filled; i++) { + FUZZ_ASSERT(bounds[i].compressedSize > 0); + FUZZ_ASSERT(bounds[i].offset <= size); + FUZZ_ASSERT(bounds[i].compressedSize <= size - bounds[i].offset); + if (i == 0) { + FUZZ_ASSERT(bounds[i].offset == 0); + } else { + FUZZ_ASSERT(bounds[i].offset == + bounds[i-1].offset + bounds[i-1].compressedSize); + } + } + if (filled > 0) { + FUZZ_ASSERT(bounds[filled-1].offset + bounds[filled-1].compressedSize == size); + } else { + FUZZ_ASSERT(size == 0); + } + } else if (!ZSTD_isError(counted)) { + /* counting pass succeeded: the fill pass may only fail on capacity */ + FUZZ_ASSERT(ZSTD_getErrorCode(filled) == ZSTD_error_dstSize_tooSmall); + } else { + /* counting pass failed: the fill pass must also fail (it walks the + * same frames; it may hit dstSize_tooSmall before the bad frame). */ + FUZZ_ASSERT(ZSTD_isError(filled)); + } + } return 0; }