From 38fc082fe0af26d9f4c39d99b9c8d3f642e98a83 Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:02:34 +0530 Subject: [PATCH] chore: finish the CI cleanup round - perf-regression.yml installs with -c constraints/ci.txt like the other gating jobs (left out of #194 to avoid conflicting with #193). - MissForest convergence converts with to_numpy(dtype="float64", na_value=np.nan): the same pandas < 2 masked-array conversion that #192 fixed in the streaming state. Adds a nullable Int16 MissForest test. - contributor-roadmap.md no longer advertises #33 and #31, which are resolved; it points at the good-first-issue label instead. --- .github/workflows/perf-regression.yml | 2 +- docs/community/contributor-roadmap.md | 8 +++----- src/freshdata/imputation/missforest.py | 5 ++++- tests/test_missforest_imputation.py | 13 +++++++++++++ 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/perf-regression.yml b/.github/workflows/perf-regression.yml index 64ef21a9..dcc7b811 100644 --- a/.github/workflows/perf-regression.yml +++ b/.github/workflows/perf-regression.yml @@ -54,7 +54,7 @@ jobs: - name: Install run: | python -m pip install -U pip - pip install -e ".[bench]" + pip install -c constraints/ci.txt -e ".[bench]" - name: Resolve base commit id: base env: diff --git a/docs/community/contributor-roadmap.md b/docs/community/contributor-roadmap.md index c6146a4b..604d4bec 100644 --- a/docs/community/contributor-roadmap.md +++ b/docs/community/contributor-roadmap.md @@ -35,11 +35,9 @@ review does not depend on understanding the whole cleaning pipeline. [Great Expectations recipe (#8)](https://github.com/FreshCode-Org/freshdata/issues/8), [ydata-profiling comparison (#7)](https://github.com/FreshCode-Org/freshdata/issues/7). Files live in [`examples/`](https://github.com/FreshCode-Org/freshdata/tree/main/examples). -- **Small isolated cleanups** — - [de-duplicate `_has_outliers` (#33)](https://github.com/FreshCode-Org/freshdata/issues/33) - and - [fix boolean coercion in `_coerce_series` (#31)](https://github.com/FreshCode-Org/freshdata/issues/31) - are both labeled `good first issue` and name the exact function and file. +- **Small isolated cleanups** — look for open issues labeled + [`good first issue`](https://github.com/FreshCode-Org/freshdata/labels/good%20first%20issue); + each names the exact function and file. **Skills:** Python, pandas basics. **You'll touch:** `examples/`, `docs/`, or one named module + its test. diff --git a/src/freshdata/imputation/missforest.py b/src/freshdata/imputation/missforest.py index 9aae0c1a..d6ebec36 100644 --- a/src/freshdata/imputation/missforest.py +++ b/src/freshdata/imputation/missforest.py @@ -285,7 +285,10 @@ def _convergence_delta( if plan.model_type == "regressor": prev_num = pd.to_numeric(prev, errors="coerce") cur_num = pd.to_numeric(cur, errors="coerce") - denom = float(np.nanstd(cur_num.to_numpy(dtype="float64"))) or 1.0 + # na_value: nullable (masked) columns with missing cells refuse a + # plain float64 conversion on pandas < 2. + values = cur_num.to_numpy(dtype="float64", na_value=np.nan) + denom = float(np.nanstd(values)) or 1.0 deltas.append(float(np.nanmean(np.abs(cur_num - prev_num))) / denom) else: deltas.append(float((cur.astype(object) != prev.astype(object)).mean())) diff --git a/tests/test_missforest_imputation.py b/tests/test_missforest_imputation.py index 08cb90de..8d5703e7 100644 --- a/tests/test_missforest_imputation.py +++ b/tests/test_missforest_imputation.py @@ -227,3 +227,16 @@ def test_missforest_auto_indicator_uses_prefill_informative_missingness(): assert "age_was_missing" in out.columns assert int(out["age_was_missing"].sum()) == 40 + + +def test_missforest_handles_nullable_integer_column_with_missing_values(): + # Regression: pandas < 2 refuses a plain float64 conversion of a masked + # (nullable) integer column that still holds missing values. + df = _mixed_frame() + df["age"] = pd.array(df["age"].round().astype(int), dtype="Int16") + df.loc[5:14, "age"] = pd.NA + + out, report = fd.clean(df, impute="missforest", return_report=True, **ISOLATE) + + assert out["age"].isna().sum() == 0 + assert _missforest_actions(report, "age")