fix(polars): keep integers exact when a polars column holds nulls (#467)

pl.DataFrame.to_pandas() renders an integer column that has nulls as float64,
because a NumPy integer array cannot hold one. That silently rounds every
value a float64 cannot represent: 2**53 + 1 came back as 2**53, and a UInt64
beyond 2**63 lost far more.

This is not limited to engine="polars". Every public entry point reads a
polars source through adapters.polars.to_pandas(), so fd.clean(pl_df) on the
DEFAULT engine already returned the rounded value.

Integer columns that hold nulls are now rebuilt from the raw integers plus a
null mask, giving the pandas nullable dtype of the same width (Int64, UInt64,
Int32, ...) — lossless, and what the same data already looked like when passed
in as pandas. Columns without nulls round-trip exactly today and are untouched.
The native polars engine converts its result through the same adapter instead
of calling frame.to_pandas() directly.

Default-output change: a polars input whose integer column holds nulls now
cleans as a nullable integer column instead of float64.

Closes #444

Co-authored-by: Kevin Costner <kiran.gangalakunta@gmail.com>

Author: kevincostner17

CHANGELOG.md

@@ -20,6 +20,17 @@ adheres to [Semantic Versioning](https://semver.org/).
   Polars returned a period as its raw int64 ordinal (`2020-01` → `600`) and an
   interval as a `{left, right}` struct. Both engines now fall back to the
   pandas reference for those dtypes, with the reason recorded.
+- A Polars frame whose integer column holds nulls no longer loses large
+  integers. `pl.DataFrame.to_pandas()` renders such a column as `float64`,
+  which rounds every value a float64 cannot represent, so `fd.clean(pl_df)`
+  returned `9007199254740992` for an input of `2**53 + 1` — on the **default**
+  engine, because every public entry point reads a Polars source through this
+  conversion. Those columns are now rebuilt from the raw integers plus a null
+  mask, giving the pandas nullable dtype of the same width (`Int64`, `UInt64`,
+  `Int32`, …). Integer columns without nulls are untouched. **Default-output
+  change:** a Polars input whose integer column has nulls now cleans as a
+  nullable integer column instead of `float64`, exactly as the same data
+  already did when passed in as pandas.
 - `CleanReport.revert()` no longer writes a restored value into other rows
   that share a duplicate index label. The undo log now records positional
   offsets and revert restores by position, so a frame with a non-unique index

src/freshdata/adapters/polars.py

@@ -4,10 +4,16 @@
 
 from typing import Any
 
+import numpy as np
 import pandas as pd
 
 _POLARS: Any = None
 
+#: Polars integer dtypes that map to a pandas nullable integer dtype of the same width.
+_NULLABLE_INT_DTYPES = frozenset(
+    {"Int8", "Int16", "Int32", "Int64", "UInt8", "UInt16", "UInt32", "UInt64"}
+)
+
 
 def _polars_module():
     global _POLARS
@@ -49,16 +55,41 @@ def to_pandas(df: object) -> pd.DataFrame:
     if is_polars_frame(df):
         pl_df: Any = df
         try:
-            return pl_df.to_pandas()
+            converted = pl_df.to_pandas()
         except ModuleNotFoundError as exc:
             raise ModuleNotFoundError(
                 "converting a Polars frame to pandas requires pyarrow; "
                 'install it with `pip install "freshdata-cleaner[polars]"` '
                 "(or `pip install pyarrow`)"
             ) from exc
+        return restore_nullable_integers(pl_df, converted)
     raise TypeError(f"expected pandas or polars DataFrame, got {type(df).__name__}")
 
 
+def restore_nullable_integers(pl_df: Any, converted: pd.DataFrame) -> pd.DataFrame:
+    """Rebuild integer columns that hold nulls as pandas nullable integers.
+
+    ``pl.DataFrame.to_pandas()`` renders an integer column that has nulls as
+    ``float64``, because NumPy integers cannot hold one. That silently rounds
+    every value a float64 cannot represent — ``2**53 + 1`` comes back as
+    ``2**53``, and a ``UInt64`` beyond ``2**63`` loses far more. The integers
+    are exact in polars, so rebuild those columns from the raw values plus a
+    null mask, which is both lossless and what the same data would be in pandas
+    to begin with. Columns without nulls already round-trip exactly and are
+    left alone.
+    """
+    for name, dtype in zip(pl_df.columns, pl_df.dtypes):
+        if str(dtype) not in _NULLABLE_INT_DTYPES:
+            continue
+        column = pl_df[name]
+        if column.null_count() == 0:
+            continue  # to_pandas() already gave us the exact NumPy integers
+        values = np.asarray(column.fill_null(0).to_numpy())
+        mask = np.asarray(column.is_null().to_numpy(), dtype=bool)
+        converted[name] = pd.arrays.IntegerArray(values, mask)
+    return converted
+
+
 def from_pandas(df: pd.DataFrame, original: object | None = None) -> object:
     if original is None or isinstance(original, pd.DataFrame):
         return df

src/freshdata/execution/__init__.py

@@ -124,7 +124,11 @@ def _convert_output(frame: Any, output_format: str, report: Any = None) -> Any:
     if output_format == "pandas":
         if is_pandas:
             return frame
-        return frame.to_pandas()
+        from ..adapters.polars import to_pandas as polars_to_pandas
+
+        # Not frame.to_pandas(): that renders an integer column holding nulls as
+        # float64, which rounds values a float64 cannot represent.
+        return polars_to_pandas(frame)
 
     if output_format == "polars":
         from ._lazy import require_polars

tests/test_polars_adapter.py

@@ -56,3 +56,53 @@ def test_lazy_frame_round_trips_through_default_clean():
     out = fd.clean(lf, verbose=False)
     assert isinstance(out, pl.LazyFrame)  # LazyFrame in, LazyFrame out
     assert out.collect()["b"].to_list() == ["x", "y", "z"]
+
+
+# -- #444 integer columns holding nulls must not go through float64 ------------
+
+
+@pytest.mark.parametrize(
+    ("values", "dtype", "expected"),
+    [
+        ([2**53 + 1, None, 7], pl.Int64, "Int64"),
+        ([2**63 + 1, None, 7], pl.UInt64, "UInt64"),
+        ([5, None, 7], pl.Int32, "Int32"),
+    ],
+)
+def test_to_pandas_keeps_integers_exact_when_the_column_has_nulls(values, dtype, expected):
+    """pl.to_pandas() renders these as float64, rounding past 2**53 (#444)."""
+    out = to_pandas(pl.DataFrame({"v": pl.Series(values, dtype=dtype)}))
+    assert str(out["v"].dtype) == expected
+    assert out["v"].tolist()[0] == values[0]
+    assert out["v"].isna().tolist() == [False, True, False]
+
+
+def test_to_pandas_leaves_integer_columns_without_nulls_alone():
+    out = to_pandas(pl.DataFrame({"v": pl.Series([2**53 + 1, 3], dtype=pl.Int64)}))
+    assert str(out["v"].dtype) == "int64"
+    assert out["v"].tolist() == [2**53 + 1, 3]
+
+
+def test_clean_does_not_round_large_integers_from_a_polars_frame():
+    """The default engine reads a polars source through the same adapter (#444)."""
+    df = pl.DataFrame({"v": pl.Series([2**53 + 1, None, 7], dtype=pl.Int64), "k": [1.0, 2.0, 3.0]})
+    out = fd.clean(df, verbose=False)  # polars in, polars out
+    assert out.schema["v"] == pl.Int64
+    assert out["v"].to_list() == [2**53 + 1, None, 7]
+
+
+def test_native_polars_engine_returns_exact_integers():
+    """The native path converts its result with the same adapter (#444)."""
+    pytest.importorskip("polars")
+    df = pd.DataFrame({"v": pd.array([2**53 + 1, None, 7], dtype="Int64"), "k": [1.0, 2.0, 3.0]})
+    out, report = fd.clean(
+        df,
+        strategy="conservative",
+        fix_dtypes=False,
+        verbose=False,
+        engine="polars",
+        return_report=True,
+    )
+    assert report.backend == "polars"
+    assert str(out["v"].dtype) == "Int64"
+    assert out["v"].tolist()[0] == 2**53 + 1