Skip to content

GH-31464: [C++][Python] Fix string filters with empty value sets #50882

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
fallenmi wants to merge 1 commit into
base: main
Choose a base branch
from
+26 −0
Draft

GH-31464: [C++][Python] Fix string filters with empty value sets #50882

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cpp/src/arrow/compute/kernels/scalar_set_lookup.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,8 @@ struct InitStateVisitor {
ty1, " and ", ty2);
}
} else if ((arg_type.id() == Type::STRING || arg_type.id() == Type::LARGE_STRING) &&
!(options.value_set.type()->id() == Type::NA &&
options.value_set.length() == 0) &&
!is_base_binary_like(options.value_set.type()->id())) {
// This is a bit of a hack, but don't implicitly cast from a non-binary
// type to string, since most types support casting to string and that
Expand Down
13 changes: 13 additions & 0 deletions cpp/src/arrow/compute/kernels/scalar_set_lookup_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,12 @@ TEST_F(TestIsInKernel, ImplicitlyCastValueSet) {
ArrayFromJSON(utf8(), R"(["aaa", "bbb"])"),
"[true, true, false, false, true]");

// An empty null value set can be safely cast to the input string type.
CheckIsIn(ArrayFromJSON(utf8(), R"(["aaa", "bbb"])"), ArrayFromJSON(null(), "[]"),
"[false, false]");
CheckIsIn(ArrayFromJSON(large_utf8(), R"(["aaa", "bbb"])"), ArrayFromJSON(null(), "[]"),
"[false, false]");

// But explicitly deny implicit casts from non-binary to utf8 to
// avoid surprises
ASSERT_RAISES(TypeError,
Expand Down Expand Up @@ -1455,6 +1461,13 @@ TEST_F(TestIndexInKernel, ImplicitlyCastValueSet) {
ArrayFromJSON(large_utf8(), R"(["aaa", "bbb"])"), "[0, 1, null, null, 1]");
CheckIndexIn(ArrayFromJSON(large_utf8(), R"(["aaa", "bbb", "ccc", null, "bbb"])"),
ArrayFromJSON(utf8(), R"(["aaa", "bbb"])"), "[0, 1, null, null, 1]");

// An empty null value set can be safely cast to the input string type.
CheckIndexIn(ArrayFromJSON(utf8(), R"(["aaa", "bbb"])"), ArrayFromJSON(null(), "[]"),
"[null, null]");
CheckIndexIn(ArrayFromJSON(large_utf8(), R"(["aaa", "bbb"])"),
ArrayFromJSON(null(), "[]"), "[null, null]");

// But explicitly deny implicit casts from non-binary to utf8 to
// avoid surprises
ASSERT_RAISES(TypeError,
Expand Down
11 changes: 11 additions & 0 deletions python/pyarrow/tests/test_compute.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1730,6 +1730,17 @@ def test_filter_table():
assert result.equals(expected_null)


@pytest.mark.parametrize("dictionary_encode", [False, True],
ids=["string", "dictionary-string"])
def test_filter_table_expression_empty_set(dictionary_encode):
values = pa.array(["a", "b"])
if dictionary_encode:
values = values.dictionary_encode()

table = pa.table({"a": values})
assert table.filter(pc.field("a").isin([])).equals(table.slice(0, 0))


def test_filter_errors():
arr = pa.chunked_array([["a", None], ["c", "d", "e"]])
batch = pa.record_batch(
Expand Down