feat: add fd.clean_excel() companion to fd.clean_csv()

CHANGELOG.md

@@ -7,6 +7,9 @@ adheres to [Semantic Versioning](https://semver.org/).
 ## [Unreleased]
 
 ### Added
+- `fd.clean_excel()`, the Excel companion to `fd.clean_csv()`: reads one sheet,
+  cleans it, and optionally writes the result, with formula sanitization on by
+  default. Needs the new `excel` extra (`openpyxl`).
 - `CleanReport.to_json()` and `CleanReport.write_json()` for first-class audit
   report serialization without manual `json.dumps(...)` calls.
 - Added a runnable PyJanitor interoperability example that demonstrates both

benchmarks/truthbench/inventory.py

@@ -102,6 +102,7 @@ def __post_init__(self) -> None:
     "clean_csv",
     "clean_domain_file",
     "clean_enterprise",
+    "clean_excel",
     "clean_timeseries",
     "cluster_column",
     "compare_clean",
@@ -206,6 +207,7 @@ def discover_public_names() -> tuple[str, ...]:
     "clean_csv": "cleaning",
     "clean_domain_file": "cleaning",
     "clean_enterprise": "cleaning",
+    "clean_excel": "cleaning",
     "clean_timeseries": "cleaning",
     "compare_clean": "cleaning",
     "compare_plans": "cleaning",

benchmarks/truthbench/surfaces/cleaning.py

@@ -112,6 +112,14 @@ def _run(
                     path = tmp.name
             result = fd.clean_csv(Path(path), return_report=True, **options)
             return result[0], result[1], {"path": str(path)}
+        if operation == "clean_excel":
+            path = _value(context, "path")
+            if path is None:
+                with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as tmp:
+                    path = tmp.name
+                frame.to_excel(path, index=False)
+            result = fd.clean_excel(Path(path), return_report=True, **options)
+            return result[0], result[1], {"path": str(path)}
         if operation in {"pipeline", "Pipeline.run"}:
             pipe = fd.pipeline()
             steps = _value(context, "steps", ("normalize_columns",))

constraints/ci.txt

@@ -321,6 +321,8 @@ email-validator==2.3.0
     # via
     #   fastapi
     #   pydantic
+et-xmlfile==2.0.0
+    # via openpyxl
 eval-type-backport==0.4.0 ; python_full_version < '3.10'
     # via apache-airflow-core
 exceptiongroup==1.3.1 ; python_full_version < '3.11'
@@ -772,6 +774,8 @@ onnxruntime==1.24.3 ; python_full_version == '3.10.*'
     # via freshdata-cleaner
 onnxruntime==1.27.0 ; python_full_version >= '3.11'
     # via freshdata-cleaner
+openpyxl==3.1.5
+    # via freshdata-cleaner
 opentelemetry-api==1.41.1 ; python_full_version < '3.10'
     # via
     #   apache-airflow-core

docs/api-reference.md

@@ -17,6 +17,8 @@ top-level attribute of `freshdata` (e.g. `import freshdata as fd; fd.clean(...)`
 
 ::: freshdata.clean_csv
 
+::: freshdata.clean_excel
+
 ::: freshdata.Cleaner
 
 ::: freshdata.pipeline

docs/threat-model.md

@@ -26,6 +26,7 @@ now safe by default; byte-exact fidelity is the explicit opt-out:
 |---|---|---|
 | `export_review_queue` (csv) | sanitize **on** | `sanitize_formulas=False` |
 | `fd.clean_csv(output_path=...)` | sanitize **on** | `sanitize_formulas=False` |
+| `fd.clean_excel(output_path=...)` (xlsx) | sanitize **on** | `sanitize_formulas=False` |
 | `freshdata clean` / `apply-plan` CLI csv output | sanitize **on** | `--no-sanitize-formulas` |
 | streaming CLI (incl. quarantine export) | sanitize **on** | `--no-sanitize-formulas` |
 | HTML-report ledger CSV download | sanitize **on** | none (spreadsheet-bound artifact) |

pyproject.toml

@@ -151,6 +151,11 @@ all = [
     "great-tables>=0.9",
     "anywidget>=0.9",
     "ipython>=7.0",
+    "openpyxl>=3.0.7",
+]
+# fd.clean_excel reads and writes .xlsx through pandas, which needs openpyxl.
+excel = [
+    "openpyxl>=3.0.7",
 ]
 # Stronger PII detection (NER) + cryptographic format-preserving encryption.
 # The base regex/context detector and surrogate FPE need none of these.
@@ -179,6 +184,7 @@ dev = [
     "sqlalchemy>=1.4",
     "duckdb>=0.10",
     "psutil>=5.9",
+    "openpyxl>=3.0.7",
 ]
 docs = [
     "mkdocs-material>=9.5",

src/freshdata/__init__.py

@@ -28,6 +28,7 @@
     clean,
     clean_csv,
     clean_domain_file,
+    clean_excel,
     clean_timeseries,
     compile_context,
     export,
@@ -177,6 +178,7 @@
     "Pipeline",
     "clean_csv",
     "clean_domain_file",
+    "clean_excel",
     "clean_timeseries",
     "compare_clean",
     "compare_plans",

src/freshdata/api.py

@@ -519,6 +519,92 @@ def clean_csv(
     return result
 
 
+def clean_excel(
+    path: str | Path,
+    config: CleanConfig | Mapping[str, object] | None = None,
+    *,
+    output_path: str | Path | None = None,
+    return_report: bool = False,
+    read_excel_kwargs: dict[str, object] | None = None,
+    to_excel_kwargs: dict[str, object] | None = None,
+    context: str | None = None,
+    policy: object | None = None,
+    strict: bool = False,
+    profile: object | None = None,
+    sanitize_formulas: bool = True,
+    **options: object,
+) -> pd.DataFrame | tuple[pd.DataFrame, CleanReport]:
+    """Read one Excel sheet, clean it, and optionally write the result to disk.
+
+    The Excel companion to :func:`clean_csv`, with the same options. Reading
+    and writing ``.xlsx`` needs ``openpyxl`` (``pip install
+    "freshdata-cleaner[excel]"``).
+
+    Parameters
+    ----------
+    path:
+        Path to the input workbook.
+    output_path:
+        Optional path to write the cleaned workbook.
+    sanitize_formulas:
+        On by default (safe by default): string cells (and column labels)
+        in the **written** workbook that start with ``= + - @ <tab> <cr>`` —
+        including after leading whitespace — are prefixed with ``'``.
+        Without it, a value such as ``=1+1`` is stored as a live formula
+        cell. Pass ``sanitize_formulas=False`` to write values unchanged;
+        the returned DataFrame is never altered either way.
+    return_report:
+        If True, return ``(cleaned_df, CleanReport)``.
+    read_excel_kwargs:
+        Optional keyword arguments forwarded to ``pandas.read_excel``. The
+        first sheet is read unless ``sheet_name`` selects another; a
+        ``sheet_name`` that selects several sheets raises ``TypeError``.
+    to_excel_kwargs:
+        Optional keyword arguments forwarded to ``DataFrame.to_excel``.
+        ``index`` defaults to False unless explicitly overridden.
+    context / policy / strict:
+        Natural-language rules or a pre-compiled
+        :class:`~freshdata.ContextPolicy`, forwarded to :func:`freshdata.clean`.
+    profile:
+        A learned :class:`~freshdata.learning.LearningProfile` (or path to a
+        ``.fdprofile``), forwarded to :func:`freshdata.clean`.
+    **options:
+        Any :class:`~freshdata.CleanConfig` field accepted by
+        :func:`freshdata.clean`.
+
+    Examples
+    --------
+    >>> import freshdata as fd
+    >>> cleaned = fd.clean_excel("input.xlsx")
+    >>> fd.clean_excel("input.xlsx", output_path="cleaned.xlsx")
+    >>> cleaned, report = fd.clean_excel("input.xlsx", return_report=True)
+    >>> fd.clean_excel("input.xlsx", read_excel_kwargs={"sheet_name": "Q3"})
+    """
+    if "report" in options:
+        return_report = bool(options.pop("report"))
+    df = pd.read_excel(path, **(read_excel_kwargs or {}))
+    if isinstance(df, dict):
+        raise TypeError(
+            "clean_excel cleans a single sheet; pass "
+            "read_excel_kwargs={'sheet_name': <name or index>} to choose one"
+        )
+    result = clean(
+        df,
+        config=config,
+        return_report=return_report,
+        context=context,
+        policy=policy,
+        strict=strict,
+        profile=profile,
+        **options,  # type: ignore[arg-type]
+    )
+    cleaned_df = cast(pd.DataFrame, result[0] if return_report else result)
+    if output_path is not None:
+        to_write = sanitize_csv_formulas(cleaned_df) if sanitize_formulas else cleaned_df
+        to_write.to_excel(output_path, **{"index": False, **(to_excel_kwargs or {})})
+    return result
+
+
 def compile_context(
     text: str,
     df: pd.DataFrame | None = None,

tests/test_clean_excel.py

@@ -0,0 +1,123 @@
+"""``fd.clean_excel``: the Excel companion to ``fd.clean_csv`` (#166)."""
+
+from __future__ import annotations
+
+import inspect
+
+import pandas as pd
+import pytest
+
+import freshdata as fd
+
+openpyxl = pytest.importorskip("openpyxl")
+
+
+def _write_workbook(path, sheets):
+    """Write ``{sheet: rows}`` with openpyxl so ``=`` strings stay text cells."""
+    wb = openpyxl.Workbook()
+    wb.remove(wb.active)
+    for name, rows in sheets.items():
+        ws = wb.create_sheet(name)
+        for row in rows:
+            ws.append(row)
+        for row in ws.iter_rows():
+            for cell in row:
+                if isinstance(cell.value, str):
+                    cell.data_type = "s"
+    wb.save(path)
+    return path
+
+
+MESSY = [
+    ["Name ", "Age", "note"],
+    [" Ada", 36, "=1+1"],
+    ["Grace", None, "ok"],
+    ["Alan", 41, "@SUM(A1)"],
+]
+
+
+def _cells(path):
+    ws = openpyxl.load_workbook(path).active
+    return {c.value: c.data_type for row in ws.iter_rows() for c in row if c.value is not None}
+
+
+def test_round_trip_matches_clean(tmp_path):
+    src = _write_workbook(tmp_path / "in.xlsx", {"data": MESSY})
+    out = tmp_path / "out.xlsx"
+
+    result = fd.clean_excel(src, output_path=out, sanitize_formulas=False)
+
+    expected = fd.clean(pd.read_excel(src))
+    pd.testing.assert_frame_equal(pd.DataFrame(result), pd.DataFrame(expected))
+    assert list(result.columns) == ["name", "age", "note"]
+    written = pd.read_excel(out)
+    safe_cols = ["name", "age"]
+    pd.testing.assert_frame_equal(
+        written[safe_cols], pd.DataFrame(result)[safe_cols], check_dtype=False
+    )
+
+
+def test_return_report_and_report_alias(tmp_path):
+    src = _write_workbook(tmp_path / "in.xlsx", {"data": MESSY})
+
+    cleaned, report = fd.clean_excel(src, return_report=True)
+    assert isinstance(cleaned, pd.DataFrame)
+    assert isinstance(report, fd.CleanReport)
+
+    cleaned_alias, report_alias = fd.clean_excel(src, report=True)
+    assert isinstance(report_alias, fd.CleanReport)
+
+
+def test_sheet_selection(tmp_path):
+    src = _write_workbook(
+        tmp_path / "in.xlsx",
+        {"first": MESSY, "second": [["id", "value"], [1, "a"], [2, "b"]]},
+    )
+
+    cleaned = fd.clean_excel(src, read_excel_kwargs={"sheet_name": "second"})
+    assert list(cleaned.columns) == ["id", "value"]
+    assert len(cleaned) == 2
+
+    with pytest.raises(TypeError, match="single sheet"):
+        fd.clean_excel(src, read_excel_kwargs={"sheet_name": None})
+
+
+def test_sanitizes_formulas_by_default(tmp_path):
+    src = _write_workbook(
+        tmp_path / "in.xlsx", {"data": [["=HDR", "note"], ["x", "=1+1"], ["y", "@SUM(A1)"]]}
+    )
+    out = tmp_path / "out.xlsx"
+
+    result = fd.clean_excel(src, output_path=out)
+
+    cells = _cells(out)
+    assert cells.get("'=1+1") == "s"
+    assert cells.get("'@SUM(A1)") == "s"
+    assert not any(t == "f" for t in cells.values())
+    # the returned frame is NOT sanitized — only the written artifact is
+    assert (result["note"] == "=1+1").any()
+
+
+def test_sanitize_formulas_opt_out_writes_formula_cells(tmp_path):
+    src = _write_workbook(tmp_path / "in.xlsx", {"data": [["note"], ["=1+1"], ["ok"]]})
+    out = tmp_path / "out.xlsx"
+
+    fd.clean_excel(src, output_path=out, sanitize_formulas=False)
+
+    assert _cells(out).get("=1+1") == "f"
+
+
+def test_signature_matches_clean_csv():
+    def names(fn, renames=None):
+        renames = renames or {}
+        return [renames.get(p, p) for p in inspect.signature(fn).parameters]
+
+    assert names(fd.clean_excel) == names(
+        fd.clean_csv,
+        {"read_csv_kwargs": "read_excel_kwargs", "to_csv_kwargs": "to_excel_kwargs"},
+    )
+
+
+def test_exported():
+    assert "clean_excel" in fd.__all__
+    assert fd.clean_excel is not None