Fix DeprecationWarning in _json_value by handling array-like values; …

src/freshdata/duplicate_defense.py

@@ -17,6 +17,7 @@
 from datetime import datetime, timezone
 from typing import Any, Literal
 
+import numpy as np
 import pandas as pd
 
 from .review import ReviewDataset, ReviewQueue
@@ -320,21 +321,41 @@ def _hash_payload(payload: Any) -> str:
 
 
 def _json_value(value: Any) -> Any:
+    """Convert a Python/pandas/numpy value into JSON-friendly primitives.
+
+    This function intentionally handles array-like inputs (numpy arrays and
+    pandas Series) before calling ``pd.isna`` to avoid ambiguous truth-value
+    checks that raise DeprecationWarning in newer pandas/numpy. Scalars are
+    converted to None when missing (pd.NA/np.nan), and containers are
+    recursively converted.
+    """
     if value is None:
         return None
     if isinstance(value, (str, int, float, bool)):
         return value
+
+    # Handle array-like inputs (pandas Series, numpy arrays, lists/tuples/sets)
+    # before calling pd.isna so we don't get an array result used in a truth test.
+    try:
+        if isinstance(value, pd.Series):
+            return [_json_value(v) for v in value.tolist()]
+        if isinstance(value, np.ndarray):
+            return [_json_value(v) for v in value.tolist()]
+        if isinstance(value, (list, tuple, set)):
+            return [_json_value(v) for v in value]
+    except Exception:
+        # defensive: fall through to scalar handling
+        pass
+
+    # Now safe to call pd.isna for scalar-like objects only.
     try:
-        # Only check pd.isna() on scalar values to avoid ambiguous array evaluation
-        if not isinstance(value, (dict, list, tuple, set)):
-            result = pd.isna(value)
-            # Handle case where result is a scalar boolean
-            if isinstance(result, (bool, type(pd.NA))) and result:
-                return None
-    except (TypeError, ValueError):
+        if pd.isna(value):
+            return None
+    except Exception:
         pass
+
     if isinstance(value, dict):
         return {str(k): _json_value(v) for k, v in value.items()}
-    if isinstance(value, (list, tuple, set)):
-        return [_json_value(v) for v in value]
+
+    # Fallback: stringify unknown objects to keep deterministic output.
     return str(value)

src/freshdata/plan.py

@@ -8,6 +8,7 @@
 from dataclasses import dataclass, field
 from typing import Any
 
+import numpy as np
 import pandas as pd
 from pandas.api.types import is_bool_dtype, is_numeric_dtype
 
@@ -325,7 +326,12 @@ def _choice_dict(choice: ModelChoice | None) -> dict[str, Any] | None:
 
 
 def _json_value(value: Any) -> Any:
-    """Convert pandas/numpy scalar values into JSON-friendly Python values."""
+    """Convert pandas/numpy scalar values into JSON-friendly Python values.
+
+    Array-like inputs (numpy arrays, pandas Series) are handled before
+    calling ``pd.isna`` to avoid ambiguous truth-value checks that raise
+    DeprecationWarning in newer pandas/numpy.
+    """
     if value is None:
         return None
     if isinstance(value, dict):
@@ -334,6 +340,12 @@ def _json_value(value: Any) -> Any:
         return [_json_value(v) for v in value]
     if isinstance(value, list):
         return [_json_value(v) for v in value]
+    # Handle array-like inputs before calling pd.isna to avoid array
+    # truth-value ambiguity.
+    if isinstance(value, pd.Series):
+        return [_json_value(v) for v in value.tolist()]
+    if isinstance(value, np.ndarray):
+        return [_json_value(v) for v in value.tolist()]
     if isinstance(value, float) and math.isnan(value):
         return None
     try:
@@ -365,6 +377,8 @@ def _values_equal(left: Any, right: Any) -> bool:
 
 
 def _is_missing_scalar(value: Any) -> bool:
+    if isinstance(value, (list, tuple, dict, np.ndarray, pd.Series)):
+        return False
     try:
         return bool(pd.isna(value))
     except (TypeError, ValueError):

tests/test_json_value.py

@@ -0,0 +1,29 @@
+import numpy as np
+import pandas as pd
+
+from freshdata.duplicate_defense import _json_value
+from freshdata.plan import _is_missing_scalar
+from freshdata.plan import _json_value as plan_json_value
+
+
+def test_json_value_handles_numpy_array_and_series():
+    """Regression: duplicate_defense._json_value must not raise on arrays."""
+    assert _json_value(np.array([])) == []
+    assert _json_value(np.array([1, None, 3])) == [1, None, 3]
+    assert _json_value(pd.Series([None, "a", 2])) == [None, "a", 2]
+
+
+def test_plan_json_value_handles_numpy_array_and_series():
+    """Regression: plan._json_value must not raise on arrays."""
+    assert plan_json_value(np.array([])) == []
+    assert plan_json_value(np.array([1, None, 3])) == [1, None, 3]
+    assert plan_json_value(pd.Series([None, "a", 2])) == [None, "a", 2]
+
+
+def test_is_missing_scalar_rejects_array_like():
+    """_is_missing_scalar should return False for containers, not raise."""
+    assert _is_missing_scalar(np.array([])) is False
+    assert _is_missing_scalar(np.array([1, 2])) is False
+    assert _is_missing_scalar(pd.Series([1])) is False
+    assert _is_missing_scalar([]) is False
+    assert _is_missing_scalar({}) is False