diff --git a/src/freshdata/duplicate_defense.py b/src/freshdata/duplicate_defense.py index 0772612b..b0cc45dd 100644 --- a/src/freshdata/duplicate_defense.py +++ b/src/freshdata/duplicate_defense.py @@ -17,6 +17,7 @@ from datetime import datetime, timezone from typing import Any, Literal +import numpy as np import pandas as pd from .review import ReviewDataset, ReviewQueue @@ -320,21 +321,41 @@ def _hash_payload(payload: Any) -> str: def _json_value(value: Any) -> Any: + """Convert a Python/pandas/numpy value into JSON-friendly primitives. + + This function intentionally handles array-like inputs (numpy arrays and + pandas Series) before calling ``pd.isna`` to avoid ambiguous truth-value + checks that raise DeprecationWarning in newer pandas/numpy. Scalars are + converted to None when missing (pd.NA/np.nan), and containers are + recursively converted. + """ if value is None: return None if isinstance(value, (str, int, float, bool)): return value + + # Handle array-like inputs (pandas Series, numpy arrays, lists/tuples/sets) + # before calling pd.isna so we don't get an array result used in a truth test. + try: + if isinstance(value, pd.Series): + return [_json_value(v) for v in value.tolist()] + if isinstance(value, np.ndarray): + return [_json_value(v) for v in value.tolist()] + if isinstance(value, (list, tuple, set)): + return [_json_value(v) for v in value] + except Exception: + # defensive: fall through to scalar handling + pass + + # Now safe to call pd.isna for scalar-like objects only. try: - # Only check pd.isna() on scalar values to avoid ambiguous array evaluation - if not isinstance(value, (dict, list, tuple, set)): - result = pd.isna(value) - # Handle case where result is a scalar boolean - if isinstance(result, (bool, type(pd.NA))) and result: - return None - except (TypeError, ValueError): + if pd.isna(value): + return None + except Exception: pass + if isinstance(value, dict): return {str(k): _json_value(v) for k, v in value.items()} - if isinstance(value, (list, tuple, set)): - return [_json_value(v) for v in value] + + # Fallback: stringify unknown objects to keep deterministic output. return str(value) diff --git a/src/freshdata/plan.py b/src/freshdata/plan.py index 5d3512d8..28b48f40 100644 --- a/src/freshdata/plan.py +++ b/src/freshdata/plan.py @@ -8,6 +8,7 @@ from dataclasses import dataclass, field from typing import Any +import numpy as np import pandas as pd from pandas.api.types import is_bool_dtype, is_numeric_dtype @@ -325,7 +326,12 @@ def _choice_dict(choice: ModelChoice | None) -> dict[str, Any] | None: def _json_value(value: Any) -> Any: - """Convert pandas/numpy scalar values into JSON-friendly Python values.""" + """Convert pandas/numpy scalar values into JSON-friendly Python values. + + Array-like inputs (numpy arrays, pandas Series) are handled before + calling ``pd.isna`` to avoid ambiguous truth-value checks that raise + DeprecationWarning in newer pandas/numpy. + """ if value is None: return None if isinstance(value, dict): @@ -334,6 +340,12 @@ def _json_value(value: Any) -> Any: return [_json_value(v) for v in value] if isinstance(value, list): return [_json_value(v) for v in value] + # Handle array-like inputs before calling pd.isna to avoid array + # truth-value ambiguity. + if isinstance(value, pd.Series): + return [_json_value(v) for v in value.tolist()] + if isinstance(value, np.ndarray): + return [_json_value(v) for v in value.tolist()] if isinstance(value, float) and math.isnan(value): return None try: @@ -365,6 +377,8 @@ def _values_equal(left: Any, right: Any) -> bool: def _is_missing_scalar(value: Any) -> bool: + if isinstance(value, (list, tuple, dict, np.ndarray, pd.Series)): + return False try: return bool(pd.isna(value)) except (TypeError, ValueError): diff --git a/tests/test_json_value.py b/tests/test_json_value.py new file mode 100644 index 00000000..24ede9b8 --- /dev/null +++ b/tests/test_json_value.py @@ -0,0 +1,29 @@ +import numpy as np +import pandas as pd + +from freshdata.duplicate_defense import _json_value +from freshdata.plan import _is_missing_scalar +from freshdata.plan import _json_value as plan_json_value + + +def test_json_value_handles_numpy_array_and_series(): + """Regression: duplicate_defense._json_value must not raise on arrays.""" + assert _json_value(np.array([])) == [] + assert _json_value(np.array([1, None, 3])) == [1, None, 3] + assert _json_value(pd.Series([None, "a", 2])) == [None, "a", 2] + + +def test_plan_json_value_handles_numpy_array_and_series(): + """Regression: plan._json_value must not raise on arrays.""" + assert plan_json_value(np.array([])) == [] + assert plan_json_value(np.array([1, None, 3])) == [1, None, 3] + assert plan_json_value(pd.Series([None, "a", 2])) == [None, "a", 2] + + +def test_is_missing_scalar_rejects_array_like(): + """_is_missing_scalar should return False for containers, not raise.""" + assert _is_missing_scalar(np.array([])) is False + assert _is_missing_scalar(np.array([1, 2])) is False + assert _is_missing_scalar(pd.Series([1])) is False + assert _is_missing_scalar([]) is False + assert _is_missing_scalar({}) is False