Skip to content

dictBuilder: validate NULL buffer inputs and arithmetic bounds #4804

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
filtede98 wants to merge 1 commit into from
Closed

dictBuilder: validate NULL buffer inputs and arithmetic bounds #4804

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 18 additions & 8 deletions lib/dictBuilder/cover.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,9 @@ static COVER_ctx_t *g_coverCtx = NULL;
size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
size_t sum = 0;
unsigned i;
if (!samplesSizes) return 0;
for (i = 0; i < nbSamples; ++i) {
if (samplesSizes[i] > ((size_t)-1) - sum) return (size_t)-1;
sum += samplesSizes[i];
}
return sum;
Expand Down Expand Up @@ -630,14 +632,22 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
unsigned d, double splitPoint, int displayLevel)
{
const BYTE *const samples = (const BYTE *)samplesBuffer;
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
size_t totalSamplesSize;
unsigned nbTrainSamples;
unsigned nbTestSamples;
size_t trainingSamplesSize;
size_t testSamplesSize;

/* Checks */
if (!samplesBuffer || !samplesSizes) return ERROR(srcSize_wrong);

totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
/* Split samples into testing and training sets */
const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
ctx->displayLevel = displayLevel;
/* Checks */
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
Expand Down Expand Up @@ -825,7 +835,7 @@ ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
DISPLAYLEVEL(1, "Cover must have at least one input file\n");
return ERROR(srcSize_wrong);
}
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
if (dictBuffer == NULL || dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
ZDICT_DICTSIZE_MIN);
return ERROR(dstSize_tooSmall);
Expand Down Expand Up @@ -1235,7 +1245,7 @@ ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
DISPLAYLEVEL(1, "Cover must have at least one input file\n");
return ERROR(srcSize_wrong);
}
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
if (dictBuffer == NULL || dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
ZDICT_DICTSIZE_MIN);
return ERROR(dstSize_tooSmall);
Expand Down
24 changes: 16 additions & 8 deletions lib/dictBuilder/fastcover.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,15 +311,23 @@ FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx,
int displayLevel)
{
const BYTE* const samples = (const BYTE*)samplesBuffer;
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
size_t totalSamplesSize;
unsigned nbTrainSamples;
unsigned nbTestSamples;
size_t trainingSamplesSize;
size_t testSamplesSize;

/* Checks */
if (!samplesBuffer || !samplesSizes) return ERROR(srcSize_wrong);

totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
/* Split samples into testing and training sets */
const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
ctx->displayLevel = displayLevel;

/* Checks */
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) {
DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
Expand Down Expand Up @@ -572,7 +580,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n");
return ERROR(srcSize_wrong);
}
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
if (dictBuffer == NULL || dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
ZDICT_DICTSIZE_MIN);
return ERROR(dstSize_tooSmall);
Expand Down Expand Up @@ -662,7 +670,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n");
return ERROR(srcSize_wrong);
}
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
if (dictBuffer == NULL || dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
ZDICT_DICTSIZE_MIN);
return ERROR(dstSize_tooSmall);
Expand Down
14 changes: 10 additions & 4 deletions lib/dictBuilder/zdict.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,15 +101,15 @@ const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(error

unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)
{
if (dictSize < 8) return 0;
if (dictBuffer == NULL || dictSize < 8) return 0;
if (MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return 0;
return MEM_readLE32((const char*)dictBuffer + 4);
}

size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize)
{
size_t headerSize;
if (dictSize <= 8 || MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return ERROR(dictionary_corrupted);
if (dictBuffer == NULL || dictSize <= 8 || MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return ERROR(dictionary_corrupted);

{ ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
U32* wksp = (U32*)malloc(HUF_WORKSPACE_SIZE);
Expand Down Expand Up @@ -624,9 +624,13 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,

static size_t ZDICT_totalSampleSize(const size_t* fileSizes, unsigned nbFiles)
{
size_t total=0;
size_t total = 0;
unsigned u;
for (u=0; u<nbFiles; u++) total += fileSizes[u];
if (!fileSizes) return 0;
for (u = 0; u < nbFiles; u++) {
if (fileSizes[u] > ((size_t)-1) - total) return (size_t)-1;
total += fileSizes[u];
}
return total;
}

Expand Down Expand Up @@ -1092,7 +1096,9 @@ size_t ZDICT_trainFromBuffer_legacy(void* dictBuffer, size_t dictBufferCapacity,
size_t result;
void* newBuff;
size_t const sBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples);
if (!dictBuffer || !samplesBuffer || !samplesSizes) return 0;
if (sBuffSize < ZDICT_MIN_SAMPLES_SIZE) return 0; /* not enough content => no dictionary */
if (sBuffSize > ((size_t)-1) - NOISELENGTH) return ERROR(memory_allocation);

newBuff = malloc(sBuffSize + NOISELENGTH);
if (!newBuff) return ERROR(memory_allocation);
Expand Down
5 changes: 5 additions & 0 deletions tests/fuzzer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3136,6 +3136,11 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);

DISPLAYLEVEL(3, "test%3i : check dictID and headerSize on NULL buffer : ", testNb++);
if (ZDICT_getDictID(NULL, 100) != 0) goto _output_error;
if (!ZDICT_isError(ZDICT_getDictHeaderSize(NULL, 100))) goto _output_error;
DISPLAYLEVEL(3, "OK \n");

DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++);
cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize,
Expand Down