Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions cpp/src/arrow/array/array_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -767,6 +767,43 @@ TEST_F(TestArray, TestMakeArrayFromScalar) {
}
}

TEST_F(TestArray, TestMakeArrayFromScalarOffsetOverflow) {
// Regression test for GH-36388: MakeArrayFromScalar should return an error
// when the total data size would overflow 32-bit offsets instead of silently
// producing an invalid array with negative offsets.

// A single-byte string repeated 2^31 times overflows int32 offsets
auto scalar = MakeScalar("x");
int64_t length = static_cast<int64_t>(1) << 31;
ASSERT_RAISES(Invalid, MakeArrayFromScalar(*scalar, length));

// A two-byte string repeated just over INT32_MAX/2 times also overflows
auto scalar2 = MakeScalar("xy");
int64_t length2 = (static_cast<int64_t>(1) << 30) + 1;
ASSERT_RAISES(Invalid, MakeArrayFromScalar(*scalar2, length2));

// Binary type has the same issue
auto bin_scalar = std::make_shared<BinaryScalar>(Buffer::FromString("abc"));
int64_t length3 = (static_cast<int64_t>(std::numeric_limits<int32_t>::max()) / 3) + 1;
ASSERT_RAISES(Invalid, MakeArrayFromScalar(*bin_scalar, length3));

// Large string type should NOT overflow (uses 64-bit offsets)
auto large_scalar = std::make_shared<LargeStringScalar>("x");
// Just verify it doesn't raise for a small count (we can't allocate 2^31 bytes here)
ASSERT_OK_AND_ASSIGN(auto arr, MakeArrayFromScalar(*large_scalar, 16));
ASSERT_EQ(arr->length(), 16);

// A length that itself exceeds the offset type's range must be rejected too,
// even independent of the value size (e.g. an empty string repeated too many
// times to index with int32 offsets).
auto empty_scalar = std::make_shared<StringScalar>("");
int64_t length4 = static_cast<int64_t>(std::numeric_limits<int32_t>::max()) + 1;
ASSERT_RAISES(Invalid, MakeArrayFromScalar(*empty_scalar, length4));

// A negative length must be rejected outright.
ASSERT_RAISES(Invalid, MakeArrayFromScalar(*scalar, -1));
}

TEST_F(TestArray, TestMakeArrayFromScalarSliced) {
// Regression test for ARROW-13437
auto scalars = GetScalars();
Expand Down
21 changes: 21 additions & 0 deletions cpp/src/arrow/array/util.cc
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
#include "arrow/util/checked_cast.h"
#include "arrow/util/decimal.h"
#include "arrow/util/endian.h"
#include "arrow/util/int_util_overflow.h"
#include "arrow/util/logging_internal.h"
#include "arrow/util/sort_internal.h"
#include "arrow/visit_data_inline.h"
Expand All @@ -51,6 +52,7 @@
namespace arrow {

using internal::checked_cast;
using internal::MultiplyWithOverflow;

// ----------------------------------------------------------------------
// Loading from ArrayData
Expand Down Expand Up @@ -853,6 +855,22 @@ class RepeatedArrayFactory {

template <typename OffsetType>
Status CreateOffsetsBuffer(OffsetType value_length, std::shared_ptr<Buffer>* out) {
// `length_` is the repeat count and is always representable in OffsetType here:
// MakeArrayFromScalar rejects negative lengths up front, and a length that itself
// exceeds the offset type's range can never produce a valid offsets buffer.
if (length_ > static_cast<int64_t>(std::numeric_limits<OffsetType>::max())) {
return Status::Invalid("length exceeds the maximum value of offset_type: ",
length_, " is greater than ",
std::numeric_limits<OffsetType>::max());
}
// Guard against the total data size (value_length * length_) overflowing the
// offset type, which would otherwise silently wrap around and produce an
// invalid array with negative/garbage offsets.
OffsetType total_size;
if (MultiplyWithOverflow(value_length, static_cast<OffsetType>(length_),
&total_size)) {
return Status::Invalid("offset overflow in repeated array construction");
}
TypedBufferBuilder<OffsetType> builder(pool_);
RETURN_NOT_OK(builder.Resize(length_ + 1));
OffsetType offset = 0;
Expand Down Expand Up @@ -905,6 +923,9 @@ Result<std::shared_ptr<Array>> MakeArrayOfNull(const std::shared_ptr<DataType>&

Result<std::shared_ptr<Array>> MakeArrayFromScalar(const Scalar& scalar, int64_t length,
MemoryPool* pool) {
if (length < 0) {
return Status::Invalid("length cannot be negative: ", length);
}
// Null union scalars still have a type code associated
if (!scalar.is_valid && !is_union(scalar.type->id())) {
return MakeArrayOfNull(scalar.type, length, pool);
Expand Down
10 changes: 10 additions & 0 deletions python/pyarrow/tests/test_array.py
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,16 @@ def test_array_from_dictionary_scalar():
assert result.equals(expected)


def test_repeat_offset_overflow():
# GH-36388: pa.repeat should raise an error when the total data size
# would overflow 32-bit offsets, instead of returning an invalid array.
with pytest.raises(pa.ArrowInvalid, match="overflow"):
pa.repeat("x", 2**31)

with pytest.raises(pa.ArrowInvalid, match="overflow"):
pa.repeat("xy", 2**30 + 1)


def test_array_getitem():
arr = pa.array(range(10, 15))
lst = arr.to_pylist()
Expand Down