From a37c13528fecc2a8c7857845ffcbb22a4b7d435d Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Thu, 18 Jun 2026 15:38:41 +0530 Subject: [PATCH 1/2] Fix DeprecationWarning in _json_value by handling array-like values; add regression test --- src/freshdata/duplicate_defense.py | 39 +++++++++++++++++++++++------- tests/test_json_value.py | 10 ++++++++ 2 files changed, 40 insertions(+), 9 deletions(-) create mode 100644 tests/test_json_value.py diff --git a/src/freshdata/duplicate_defense.py b/src/freshdata/duplicate_defense.py index 0772612b..85e354c2 100644 --- a/src/freshdata/duplicate_defense.py +++ b/src/freshdata/duplicate_defense.py @@ -18,6 +18,7 @@ from typing import Any, Literal import pandas as pd +import numpy as np from .review import ReviewDataset, ReviewQueue @@ -320,21 +321,41 @@ def _hash_payload(payload: Any) -> str: def _json_value(value: Any) -> Any: + """Convert a Python/pandas/numpy value into JSON-friendly primitives. + + This function intentionally handles array-like inputs (numpy arrays and + pandas Series) before calling ``pd.isna`` to avoid ambiguous truth-value + checks that raise DeprecationWarning in newer pandas/numpy. Scalars are + converted to None when missing (pd.NA/np.nan), and containers are + recursively converted. + """ if value is None: return None if isinstance(value, (str, int, float, bool)): return value + + # Handle array-like inputs (pandas Series, numpy arrays, lists/tuples/sets) + # before calling pd.isna so we don't get an array result used in a truth test. + try: + if isinstance(value, pd.Series): + return [_json_value(v) for v in value.tolist()] + if isinstance(value, np.ndarray): + return [_json_value(v) for v in value.tolist()] + if isinstance(value, (list, tuple, set)): + return [_json_value(v) for v in value] + except Exception: + # defensive: fall through to scalar handling + pass + + # Now safe to call pd.isna for scalar-like objects only. try: - # Only check pd.isna() on scalar values to avoid ambiguous array evaluation - if not isinstance(value, (dict, list, tuple, set)): - result = pd.isna(value) - # Handle case where result is a scalar boolean - if isinstance(result, (bool, type(pd.NA))) and result: - return None - except (TypeError, ValueError): + if pd.isna(value): + return None + except Exception: pass + if isinstance(value, dict): return {str(k): _json_value(v) for k, v in value.items()} - if isinstance(value, (list, tuple, set)): - return [_json_value(v) for v in value] + + # Fallback: stringify unknown objects to keep deterministic output. return str(value) diff --git a/tests/test_json_value.py b/tests/test_json_value.py new file mode 100644 index 00000000..5c94cbbd --- /dev/null +++ b/tests/test_json_value.py @@ -0,0 +1,10 @@ +import numpy as np +import pandas as pd + +from freshdata.duplicate_defense import _json_value + + +def test_json_value_handles_numpy_array_and_series(): + assert _json_value(np.array([])) == [] + assert _json_value(np.array([1, None, 3])) == [1, None, 3] + assert _json_value(pd.Series([None, "a", 2])) == [None, "a", 2] From 21c5080e4dc6995ba16b29d8b5963fe7d718537f Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty <192861341+JohnnyWilson-Portfolio@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:12:32 +0530 Subject: [PATCH 2/2] fix: handle array-like values in plan._json_value and _is_missing_scalar Extend the DeprecationWarning fix to plan.py (not just duplicate_defense.py): - Add numpy array and pandas Series handling in plan._json_value before pd.isna() call to avoid ambiguous truth-value check - Guard _is_missing_scalar against array-like inputs (np.ndarray, pd.Series, list, tuple, dict) which would fail on bool(pd.isna(array)) - Fix import order in duplicate_defense.py (ruff I001) - Add regression tests for plan._json_value and _is_missing_scalar --- src/freshdata/duplicate_defense.py | 2 +- src/freshdata/plan.py | 16 +++++++++++++++- tests/test_json_value.py | 19 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/freshdata/duplicate_defense.py b/src/freshdata/duplicate_defense.py index 85e354c2..b0cc45dd 100644 --- a/src/freshdata/duplicate_defense.py +++ b/src/freshdata/duplicate_defense.py @@ -17,8 +17,8 @@ from datetime import datetime, timezone from typing import Any, Literal -import pandas as pd import numpy as np +import pandas as pd from .review import ReviewDataset, ReviewQueue diff --git a/src/freshdata/plan.py b/src/freshdata/plan.py index 5d3512d8..28b48f40 100644 --- a/src/freshdata/plan.py +++ b/src/freshdata/plan.py @@ -8,6 +8,7 @@ from dataclasses import dataclass, field from typing import Any +import numpy as np import pandas as pd from pandas.api.types import is_bool_dtype, is_numeric_dtype @@ -325,7 +326,12 @@ def _choice_dict(choice: ModelChoice | None) -> dict[str, Any] | None: def _json_value(value: Any) -> Any: - """Convert pandas/numpy scalar values into JSON-friendly Python values.""" + """Convert pandas/numpy scalar values into JSON-friendly Python values. + + Array-like inputs (numpy arrays, pandas Series) are handled before + calling ``pd.isna`` to avoid ambiguous truth-value checks that raise + DeprecationWarning in newer pandas/numpy. + """ if value is None: return None if isinstance(value, dict): @@ -334,6 +340,12 @@ def _json_value(value: Any) -> Any: return [_json_value(v) for v in value] if isinstance(value, list): return [_json_value(v) for v in value] + # Handle array-like inputs before calling pd.isna to avoid array + # truth-value ambiguity. + if isinstance(value, pd.Series): + return [_json_value(v) for v in value.tolist()] + if isinstance(value, np.ndarray): + return [_json_value(v) for v in value.tolist()] if isinstance(value, float) and math.isnan(value): return None try: @@ -365,6 +377,8 @@ def _values_equal(left: Any, right: Any) -> bool: def _is_missing_scalar(value: Any) -> bool: + if isinstance(value, (list, tuple, dict, np.ndarray, pd.Series)): + return False try: return bool(pd.isna(value)) except (TypeError, ValueError): diff --git a/tests/test_json_value.py b/tests/test_json_value.py index 5c94cbbd..24ede9b8 100644 --- a/tests/test_json_value.py +++ b/tests/test_json_value.py @@ -2,9 +2,28 @@ import pandas as pd from freshdata.duplicate_defense import _json_value +from freshdata.plan import _is_missing_scalar +from freshdata.plan import _json_value as plan_json_value def test_json_value_handles_numpy_array_and_series(): + """Regression: duplicate_defense._json_value must not raise on arrays.""" assert _json_value(np.array([])) == [] assert _json_value(np.array([1, None, 3])) == [1, None, 3] assert _json_value(pd.Series([None, "a", 2])) == [None, "a", 2] + + +def test_plan_json_value_handles_numpy_array_and_series(): + """Regression: plan._json_value must not raise on arrays.""" + assert plan_json_value(np.array([])) == [] + assert plan_json_value(np.array([1, None, 3])) == [1, None, 3] + assert plan_json_value(pd.Series([None, "a", 2])) == [None, "a", 2] + + +def test_is_missing_scalar_rejects_array_like(): + """_is_missing_scalar should return False for containers, not raise.""" + assert _is_missing_scalar(np.array([])) is False + assert _is_missing_scalar(np.array([1, 2])) is False + assert _is_missing_scalar(pd.Series([1])) is False + assert _is_missing_scalar([]) is False + assert _is_missing_scalar({}) is False