diff --git a/CHANGELOG.md b/CHANGELOG.md index 67fb564d..9bbc84df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Added +- `fd.clean_excel()`, the Excel companion to `fd.clean_csv()`: reads one sheet, + cleans it, and optionally writes the result, with formula sanitization on by + default. Needs the new `excel` extra (`openpyxl`). - `CleanReport.to_json()` and `CleanReport.write_json()` for first-class audit report serialization without manual `json.dumps(...)` calls. - Added a runnable PyJanitor interoperability example that demonstrates both diff --git a/benchmarks/truthbench/inventory.py b/benchmarks/truthbench/inventory.py index b3cbe9ed..706960cd 100644 --- a/benchmarks/truthbench/inventory.py +++ b/benchmarks/truthbench/inventory.py @@ -102,6 +102,7 @@ def __post_init__(self) -> None: "clean_csv", "clean_domain_file", "clean_enterprise", + "clean_excel", "clean_timeseries", "cluster_column", "compare_clean", @@ -206,6 +207,7 @@ def discover_public_names() -> tuple[str, ...]: "clean_csv": "cleaning", "clean_domain_file": "cleaning", "clean_enterprise": "cleaning", + "clean_excel": "cleaning", "clean_timeseries": "cleaning", "compare_clean": "cleaning", "compare_plans": "cleaning", diff --git a/benchmarks/truthbench/surfaces/cleaning.py b/benchmarks/truthbench/surfaces/cleaning.py index c111ba26..de99ea40 100644 --- a/benchmarks/truthbench/surfaces/cleaning.py +++ b/benchmarks/truthbench/surfaces/cleaning.py @@ -112,6 +112,14 @@ def _run( path = tmp.name result = fd.clean_csv(Path(path), return_report=True, **options) return result[0], result[1], {"path": str(path)} + if operation == "clean_excel": + path = _value(context, "path") + if path is None: + with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as tmp: + path = tmp.name + frame.to_excel(path, index=False) + result = fd.clean_excel(Path(path), return_report=True, **options) + return result[0], result[1], {"path": str(path)} if operation in {"pipeline", "Pipeline.run"}: pipe = fd.pipeline() steps = _value(context, "steps", ("normalize_columns",)) diff --git a/constraints/ci.txt b/constraints/ci.txt index 9a703436..5f895cfc 100644 --- a/constraints/ci.txt +++ b/constraints/ci.txt @@ -321,6 +321,8 @@ email-validator==2.3.0 # via # fastapi # pydantic +et-xmlfile==2.0.0 + # via openpyxl eval-type-backport==0.4.0 ; python_full_version < '3.10' # via apache-airflow-core exceptiongroup==1.3.1 ; python_full_version < '3.11' @@ -772,6 +774,8 @@ onnxruntime==1.24.3 ; python_full_version == '3.10.*' # via freshdata-cleaner onnxruntime==1.27.0 ; python_full_version >= '3.11' # via freshdata-cleaner +openpyxl==3.1.5 + # via freshdata-cleaner opentelemetry-api==1.41.1 ; python_full_version < '3.10' # via # apache-airflow-core diff --git a/docs/api-reference.md b/docs/api-reference.md index f8309305..8a5d2d7f 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -17,6 +17,8 @@ top-level attribute of `freshdata` (e.g. `import freshdata as fd; fd.clean(...)` ::: freshdata.clean_csv +::: freshdata.clean_excel + ::: freshdata.Cleaner ::: freshdata.pipeline diff --git a/docs/threat-model.md b/docs/threat-model.md index e712e3b4..d7a3b2c9 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -26,6 +26,7 @@ now safe by default; byte-exact fidelity is the explicit opt-out: |---|---|---| | `export_review_queue` (csv) | sanitize **on** | `sanitize_formulas=False` | | `fd.clean_csv(output_path=...)` | sanitize **on** | `sanitize_formulas=False` | +| `fd.clean_excel(output_path=...)` (xlsx) | sanitize **on** | `sanitize_formulas=False` | | `freshdata clean` / `apply-plan` CLI csv output | sanitize **on** | `--no-sanitize-formulas` | | streaming CLI (incl. quarantine export) | sanitize **on** | `--no-sanitize-formulas` | | HTML-report ledger CSV download | sanitize **on** | none (spreadsheet-bound artifact) | diff --git a/pyproject.toml b/pyproject.toml index 111db791..5b704c5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,6 +151,11 @@ all = [ "great-tables>=0.9", "anywidget>=0.9", "ipython>=7.0", + "openpyxl>=3.0.7", +] +# fd.clean_excel reads and writes .xlsx through pandas, which needs openpyxl. +excel = [ + "openpyxl>=3.0.7", ] # Stronger PII detection (NER) + cryptographic format-preserving encryption. # The base regex/context detector and surrogate FPE need none of these. @@ -179,6 +184,7 @@ dev = [ "sqlalchemy>=1.4", "duckdb>=0.10", "psutil>=5.9", + "openpyxl>=3.0.7", ] docs = [ "mkdocs-material>=9.5", diff --git a/src/freshdata/__init__.py b/src/freshdata/__init__.py index 66c9426d..12ad20d4 100644 --- a/src/freshdata/__init__.py +++ b/src/freshdata/__init__.py @@ -28,6 +28,7 @@ clean, clean_csv, clean_domain_file, + clean_excel, clean_timeseries, compile_context, export, @@ -177,6 +178,7 @@ "Pipeline", "clean_csv", "clean_domain_file", + "clean_excel", "clean_timeseries", "compare_clean", "compare_plans", diff --git a/src/freshdata/api.py b/src/freshdata/api.py index 901b0c70..22b55f0b 100644 --- a/src/freshdata/api.py +++ b/src/freshdata/api.py @@ -519,6 +519,92 @@ def clean_csv( return result +def clean_excel( + path: str | Path, + config: CleanConfig | Mapping[str, object] | None = None, + *, + output_path: str | Path | None = None, + return_report: bool = False, + read_excel_kwargs: dict[str, object] | None = None, + to_excel_kwargs: dict[str, object] | None = None, + context: str | None = None, + policy: object | None = None, + strict: bool = False, + profile: object | None = None, + sanitize_formulas: bool = True, + **options: object, +) -> pd.DataFrame | tuple[pd.DataFrame, CleanReport]: + """Read one Excel sheet, clean it, and optionally write the result to disk. + + The Excel companion to :func:`clean_csv`, with the same options. Reading + and writing ``.xlsx`` needs ``openpyxl`` (``pip install + "freshdata-cleaner[excel]"``). + + Parameters + ---------- + path: + Path to the input workbook. + output_path: + Optional path to write the cleaned workbook. + sanitize_formulas: + On by default (safe by default): string cells (and column labels) + in the **written** workbook that start with ``= + - @ `` — + including after leading whitespace — are prefixed with ``'``. + Without it, a value such as ``=1+1`` is stored as a live formula + cell. Pass ``sanitize_formulas=False`` to write values unchanged; + the returned DataFrame is never altered either way. + return_report: + If True, return ``(cleaned_df, CleanReport)``. + read_excel_kwargs: + Optional keyword arguments forwarded to ``pandas.read_excel``. The + first sheet is read unless ``sheet_name`` selects another; a + ``sheet_name`` that selects several sheets raises ``TypeError``. + to_excel_kwargs: + Optional keyword arguments forwarded to ``DataFrame.to_excel``. + ``index`` defaults to False unless explicitly overridden. + context / policy / strict: + Natural-language rules or a pre-compiled + :class:`~freshdata.ContextPolicy`, forwarded to :func:`freshdata.clean`. + profile: + A learned :class:`~freshdata.learning.LearningProfile` (or path to a + ``.fdprofile``), forwarded to :func:`freshdata.clean`. + **options: + Any :class:`~freshdata.CleanConfig` field accepted by + :func:`freshdata.clean`. + + Examples + -------- + >>> import freshdata as fd + >>> cleaned = fd.clean_excel("input.xlsx") + >>> fd.clean_excel("input.xlsx", output_path="cleaned.xlsx") + >>> cleaned, report = fd.clean_excel("input.xlsx", return_report=True) + >>> fd.clean_excel("input.xlsx", read_excel_kwargs={"sheet_name": "Q3"}) + """ + if "report" in options: + return_report = bool(options.pop("report")) + df = pd.read_excel(path, **(read_excel_kwargs or {})) + if isinstance(df, dict): + raise TypeError( + "clean_excel cleans a single sheet; pass " + "read_excel_kwargs={'sheet_name': } to choose one" + ) + result = clean( + df, + config=config, + return_report=return_report, + context=context, + policy=policy, + strict=strict, + profile=profile, + **options, # type: ignore[arg-type] + ) + cleaned_df = cast(pd.DataFrame, result[0] if return_report else result) + if output_path is not None: + to_write = sanitize_csv_formulas(cleaned_df) if sanitize_formulas else cleaned_df + to_write.to_excel(output_path, **{"index": False, **(to_excel_kwargs or {})}) + return result + + def compile_context( text: str, df: pd.DataFrame | None = None, diff --git a/tests/test_clean_excel.py b/tests/test_clean_excel.py new file mode 100644 index 00000000..14f4557b --- /dev/null +++ b/tests/test_clean_excel.py @@ -0,0 +1,123 @@ +"""``fd.clean_excel``: the Excel companion to ``fd.clean_csv`` (#166).""" + +from __future__ import annotations + +import inspect + +import pandas as pd +import pytest + +import freshdata as fd + +openpyxl = pytest.importorskip("openpyxl") + + +def _write_workbook(path, sheets): + """Write ``{sheet: rows}`` with openpyxl so ``=`` strings stay text cells.""" + wb = openpyxl.Workbook() + wb.remove(wb.active) + for name, rows in sheets.items(): + ws = wb.create_sheet(name) + for row in rows: + ws.append(row) + for row in ws.iter_rows(): + for cell in row: + if isinstance(cell.value, str): + cell.data_type = "s" + wb.save(path) + return path + + +MESSY = [ + ["Name ", "Age", "note"], + [" Ada", 36, "=1+1"], + ["Grace", None, "ok"], + ["Alan", 41, "@SUM(A1)"], +] + + +def _cells(path): + ws = openpyxl.load_workbook(path).active + return {c.value: c.data_type for row in ws.iter_rows() for c in row if c.value is not None} + + +def test_round_trip_matches_clean(tmp_path): + src = _write_workbook(tmp_path / "in.xlsx", {"data": MESSY}) + out = tmp_path / "out.xlsx" + + result = fd.clean_excel(src, output_path=out, sanitize_formulas=False) + + expected = fd.clean(pd.read_excel(src)) + pd.testing.assert_frame_equal(pd.DataFrame(result), pd.DataFrame(expected)) + assert list(result.columns) == ["name", "age", "note"] + written = pd.read_excel(out) + safe_cols = ["name", "age"] + pd.testing.assert_frame_equal( + written[safe_cols], pd.DataFrame(result)[safe_cols], check_dtype=False + ) + + +def test_return_report_and_report_alias(tmp_path): + src = _write_workbook(tmp_path / "in.xlsx", {"data": MESSY}) + + cleaned, report = fd.clean_excel(src, return_report=True) + assert isinstance(cleaned, pd.DataFrame) + assert isinstance(report, fd.CleanReport) + + cleaned_alias, report_alias = fd.clean_excel(src, report=True) + assert isinstance(report_alias, fd.CleanReport) + + +def test_sheet_selection(tmp_path): + src = _write_workbook( + tmp_path / "in.xlsx", + {"first": MESSY, "second": [["id", "value"], [1, "a"], [2, "b"]]}, + ) + + cleaned = fd.clean_excel(src, read_excel_kwargs={"sheet_name": "second"}) + assert list(cleaned.columns) == ["id", "value"] + assert len(cleaned) == 2 + + with pytest.raises(TypeError, match="single sheet"): + fd.clean_excel(src, read_excel_kwargs={"sheet_name": None}) + + +def test_sanitizes_formulas_by_default(tmp_path): + src = _write_workbook( + tmp_path / "in.xlsx", {"data": [["=HDR", "note"], ["x", "=1+1"], ["y", "@SUM(A1)"]]} + ) + out = tmp_path / "out.xlsx" + + result = fd.clean_excel(src, output_path=out) + + cells = _cells(out) + assert cells.get("'=1+1") == "s" + assert cells.get("'@SUM(A1)") == "s" + assert not any(t == "f" for t in cells.values()) + # the returned frame is NOT sanitized — only the written artifact is + assert (result["note"] == "=1+1").any() + + +def test_sanitize_formulas_opt_out_writes_formula_cells(tmp_path): + src = _write_workbook(tmp_path / "in.xlsx", {"data": [["note"], ["=1+1"], ["ok"]]}) + out = tmp_path / "out.xlsx" + + fd.clean_excel(src, output_path=out, sanitize_formulas=False) + + assert _cells(out).get("=1+1") == "f" + + +def test_signature_matches_clean_csv(): + def names(fn, renames=None): + renames = renames or {} + return [renames.get(p, p) for p in inspect.signature(fn).parameters] + + assert names(fd.clean_excel) == names( + fd.clean_csv, + {"read_csv_kwargs": "read_excel_kwargs", "to_csv_kwargs": "to_excel_kwargs"}, + ) + + +def test_exported(): + assert "clean_excel" in fd.__all__ + assert fd.clean_excel is not None diff --git a/uv.lock b/uv.lock index 93aacfbc..2f97f7ed 100644 --- a/uv.lock +++ b/uv.lock @@ -2551,6 +2551,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + [[package]] name = "eval-type-backport" version = "0.4.0" @@ -2948,6 +2957,7 @@ all = [ { name = "ipython", version = "8.39.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, { name = "ipython", version = "9.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "itables" }, + { name = "openpyxl" }, { name = "plotly" }, { name = "polars", version = "1.36.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "polars", version = "1.42.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -2996,6 +3006,7 @@ dev = [ { name = "jsonschema", version = "4.25.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "jsonschema", version = "4.26.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "mypy" }, + { name = "openpyxl" }, { name = "polars", version = "1.36.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "polars", version = "1.42.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pre-commit", version = "4.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, @@ -3040,6 +3051,9 @@ entity-resolution = [ { name = "duckdb", version = "1.4.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "duckdb", version = "1.5.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, ] +excel = [ + { name = "openpyxl" }, +] flight = [ { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "pyarrow", version = "25.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -3154,6 +3168,9 @@ requires-dist = [ { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.8" }, { name = "numpy", specifier = ">=1.21" }, { name = "onnxruntime", marker = "extra == 'semantic'", specifier = ">=1.15" }, + { name = "openpyxl", marker = "extra == 'all'", specifier = ">=3.0.7" }, + { name = "openpyxl", marker = "extra == 'dev'", specifier = ">=3.0.7" }, + { name = "openpyxl", marker = "extra == 'excel'", specifier = ">=3.0.7" }, { name = "pandas", specifier = ">=1.5,<3" }, { name = "plotly", marker = "extra == 'all'", specifier = ">=5.0" }, { name = "plotly", marker = "extra == 'notebook'", specifier = ">=5.0" }, @@ -3200,7 +3217,7 @@ requires-dist = [ { name = "tokenizers", marker = "extra == 'semantic'", specifier = ">=0.13" }, { name = "twine", marker = "extra == 'dev'", specifier = ">=5.0" }, ] -provides-extras = ["ml", "polars", "pyarrow", "duckdb", "spark", "bench", "outofcore", "cli", "rich", "viz", "notebook", "domains", "cleanlab", "semantic", "enterprise", "all", "privacy", "entity-resolution", "dev", "docs", "dagster", "airflow", "dbt", "integrations", "kafka", "flight", "freshcore"] +provides-extras = ["ml", "polars", "pyarrow", "duckdb", "spark", "bench", "outofcore", "cli", "rich", "viz", "notebook", "domains", "cleanlab", "semantic", "enterprise", "all", "excel", "privacy", "entity-resolution", "dev", "docs", "dagster", "airflow", "dbt", "integrations", "kafka", "flight", "freshcore"] [[package]] name = "fsspec" @@ -6188,6 +6205,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/f6/2bac21f722aa45d876d4a51f26bd0ef30e704068a3cd5021a5a7cd784271/onnxruntime-1.27.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:370d211e1ceeac4cd5f45301655463ac59e27cdc74d9f7aeb2d19ff4b7a76715", size = 18670781, upload-time = "2026-06-15T22:43:17.151Z" }, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.41.1"