Add synthetic financial ledger fixture with accounting negatives (#496)

src/freshdata/engine/context.py

@@ -31,6 +31,11 @@
 MIN_ROWS_FOR_ENGINE = 30
 
 _ID_NAME = re.compile(r"(?:^|[_\s])(?:id|uuid|guid|key)s?$|^(?:id|index|pk)$", re.I)
+_MONEY_NAME = re.compile(
+    r"price|amount|cost|salary|revenue|fee|balance|payment|charge|wage|income|"
+    r"spend|budget|paid|total|usd|eur|inr|gbp|cash|fare",
+    re.I,
+)
 _TARGET_NAMES = frozenset({"target", "label", "y", "outcome", "class", "response"})
 _TARGET_EXACT = frozenset({
     "aqi", "score", "rating", "churn", "default", "conversion", "label",
@@ -134,7 +139,9 @@ def infer_role(
     label = str(name)
     if _is_target_name(label, config):
         return "target"
-    if label in config.id_columns or _ID_NAME.search(label):
+    if label in config.id_columns or (
+        _ID_NAME.search(label) and not _MONEY_NAME.search(label)
+    ):
         return "id"
     if is_datetime64_any_dtype(s):
         return "datetime"
@@ -158,7 +165,12 @@ def infer_role(
             # Free text first: all-unique multi-word strings are prose, not keys.
             if _looks_like_text(s, nunique, non_null):
                 return "text"
-            if nunique is not None and non_null >= 20 and nunique == non_null:
+            if (
+                nunique is not None
+                and non_null >= 20
+                and nunique == non_null
+                and not _MONEY_NAME.search(label)
+            ):
                 return "id"
         return "categorical"
     # Mixed/object payloads we cannot reason about: treat as text (hands off).

src/freshdata/semantic/context.py

@@ -148,7 +148,11 @@ def _build_info(
             and (
                 ctx.role == "id"
                 or column_name_is_identifier(name)
-                or (ctx.high_cardinality and ctx.role in ("text", "categorical"))
+                or (
+                    ctx.high_cardinality
+                    and ctx.role in ("text", "categorical")
+                    and not bool(_MONEY_NAME.search(name))
+                )
             )
         )
     )
@@ -173,8 +177,7 @@ def _build_info(
         )
     )
     money_like = (
-        not free_text
-        and not identifier_like
+        not identifier_like
         and (
             semantic_type in ("money", "currency")
             or bool(_MONEY_NAME.search(name))

src/freshdata/semantic/experts.py

@@ -219,13 +219,19 @@ def parse_currency_parts(text: str) -> tuple[float | None, bool]:
     ``"1,200"`` is left to ordinary dtype repair, not treated as money.
     """
     s = text.strip()
+    accounting_negative = s.startswith("(") and s.endswith(")")
+    if accounting_negative:
+        s = s[1:-1].strip()
     has_symbol = any(c in s for c in _CURRENCY_SYMBOLS)
     codes = {t.lower() for t in re.findall(r"[A-Za-z]+", s)}
     has_code = bool(codes & _CURRENCY_CODES)
-    if not (has_symbol or has_code):
+    if not (has_symbol or has_code or accounting_negative):
         return None, False
     body = re.sub(r"[A-Za-z$€£¥₹\s\u00a0\u202f']", "", s)
-    return _split_amount(body, detect_currency(s))
+    value, ambiguous = _split_amount(body, detect_currency(s))
+    if value is not None and accounting_negative:
+        value = -value
+    return value, ambiguous
 
 
 def parse_currency(text: str) -> float | None:

tests/expectations.py

@@ -39,6 +39,7 @@
     "large_panel",
     "duplicate_heavy",
     "locale_numbers",
+    "financial_ledger",
     "mixed_roles",
 ]
 

tests/fixtures/financial_ledger.csv

@@ -0,0 +1,26 @@
+transaction_id,transaction_date,description,amount,currency,account
+TX-001,2026-01-02,Opening balance,"$1,250.00",USD,Cash
+TX-002,2026-01-03,Vendor invoice,"($1,250.00)",USD,Accounts Payable
+TX-003,2026-01-04,European sale,"EUR 1.250,50",EUR,Revenue
+TX-004,2026-01-05,European refund,"(EUR 1.250,50)",EUR,Refunds
+TX-005,2026-01-06,Small card fee," € 0,50 ",EUR,Bank Fees
+TX-006,2026-01-07,Service charge,"EUR 12,5",EUR,Bank Fees
+TX-007,2026-01-08,Payroll batch,"$2,500.00",USD,Payroll
+TX-008,2026-01-09,Payroll reversal," ($2,500.00) ",USD,Payroll
+TX-009,2026-01-10,Office rent,"EUR 2.100,00",EUR,Operating Expense
+TX-010,2026-01-11,Rent correction,"(EUR 2.100,00)",EUR,Operating Expense
+TX-011,2026-01-12,Cloud hosting,"$3,450.75",USD,Technology
+TX-012,2026-01-13,Cloud credit,"($3,450.75)",USD,Technology
+TX-013,2026-01-14,Consulting income,"EUR 4.750,25",EUR,Revenue
+TX-014,2026-01-15,Consulting reversal,"(EUR 4.750,25)",EUR,Revenue
+TX-015,2026-01-16,Travel advance," $875.00 ",USD,Travel
+TX-016,2026-01-17,Travel return," ($875.00) ",USD,Travel
+TX-017,2026-01-18,Equipment purchase,"$12,000.00",USD,Fixed Assets
+TX-018,2026-01-19,Equipment refund,"($1,200.00)",USD,Fixed Assets
+TX-019,2026-01-20,Insurance premium,"EUR 1.050,00",EUR,Insurance
+TX-020,2026-01-21,Insurance rebate,"(EUR 150,00)",EUR,Insurance
+TX-021,2026-01-22,Interest received,"$45.67",USD,Interest
+TX-022,2026-01-23,Interest correction,"($5.67)",USD,Interest
+TX-023,2026-01-24,Settlement received,"EUR 10.000,00",EUR,Settlements
+TX-024,2026-01-25,Settlement adjustment,"(EUR 250,00)",EUR,Settlements
+TX-025,2026-01-26,Unmarked accounting adjustment,"(1,250.00)",USD,Settlements

tests/fixtures/financial_ledger.expectations.json

@@ -0,0 +1,24 @@
+{
+  "balanced": {
+    "idempotent": true,
+    "row_count": 25
+  },
+  "semantic_auto": {
+    "required_conversions": {
+      "amount": "float64",
+      "transaction_date": "datetime64"
+    },
+    "target_values": {
+      "TX-002": -1250.0,
+      "TX-003": 1250.5,
+      "TX-004": -1250.5,
+      "TX-005": 0.5,
+      "TX-008": -2500.0,
+      "TX-023": 10000.0,
+      "TX-024": -250.0,
+      "TX-025": -1250.0
+    },
+    "row_count": 25,
+    "idempotent": true
+  }
+}

tests/fixtures/golden/financial_ledger.balanced.report.json

@@ -0,0 +1,45 @@
+{
+  "actions": [
+    {
+      "column": "amount",
+      "confidence": 1.0,
+      "count": 4,
+      "description": "trimmed surrounding whitespace",
+      "human_review": false,
+      "memory_influenced": false,
+      "model_id": "",
+      "rationale": "",
+      "reversible": null,
+      "risk": "low",
+      "status": "automatic",
+      "step": "strip_whitespace"
+    },
+    {
+      "column": "transaction_date",
+      "confidence": 1.0,
+      "count": 25,
+      "description": "converted to datetime64[ns]",
+      "human_review": false,
+      "memory_influenced": false,
+      "model_id": "",
+      "rationale": "",
+      "reversible": null,
+      "risk": "low",
+      "status": "automatic",
+      "step": "fix_dtypes"
+    }
+  ],
+  "cols_after": 6,
+  "cols_before": 6,
+  "columns_dropped": [],
+  "columns_imputed": [],
+  "columns_preserved": [],
+  "duplicates_removed": 0,
+  "missing_after": 0,
+  "missing_before": 0,
+  "outliers_handled": 0,
+  "recommendations": [],
+  "rows_after": 25,
+  "rows_before": 25,
+  "warnings": []
+}

tests/fixtures/golden_diff_summary.jsonl

@@ -41,3 +41,4 @@
 {"changed": false, "created": false, "fixture": "weather_json", "new_action_count": 3, "online": true, "previous_action_count": 3, "strategy": "balanced"}
 {"changed": true, "created": false, "fixture": "wine_quality", "new_action_count": 13, "online": true, "previous_action_count": 13, "strategy": "balanced"}
 {"changed": true, "created": false, "fixture": "adult_income", "new_action_count": 11, "online": true, "previous_action_count": 10, "strategy": "balanced"}
+{"changed": true, "created": true, "fixture": "financial_ledger", "new_action_count": 2, "online": false, "previous_action_count": 0, "strategy": "balanced"}

tests/test_currency_locale.py

@@ -11,12 +11,17 @@
 
 from __future__ import annotations
 
+import json
+from pathlib import Path
+
 import pandas as pd
 import pytest
 
 import freshdata as fd
 from freshdata.semantic.experts import parse_currency, parse_currency_parts
 
+FIXTURES_DIR = Path(__file__).parent / "fixtures"
+
 # -- European formats are no longer read as US ------------------------------
 
 
@@ -113,3 +118,22 @@ def test_clean_does_not_scale_european_amounts(text, expected):
     )
     out = fd.clean(df, verbose=False, semantic_mode="auto")
     assert out["amount"].iloc[8] == expected
+
+
+def test_clean_financial_ledger_fixture_respects_locale_and_accounting_values():
+    fixture = pd.read_csv(FIXTURES_DIR / "financial_ledger.csv")
+    expectations = json.loads(
+        (FIXTURES_DIR / "financial_ledger.expectations.json").read_text()
+    )["semantic_auto"]
+
+    cleaned = fd.clean(
+        fixture, strategy="balanced", semantic_mode="auto", verbose=False
+    )
+
+    assert len(cleaned) == expectations["row_count"]
+    for column, dtype in expectations["required_conversions"].items():
+        assert str(cleaned[column].dtype).startswith(dtype)
+    for transaction_id, expected in expectations["target_values"].items():
+        actual = cleaned.loc[cleaned["transaction_id"] == transaction_id, "amount"]
+        assert len(actual) == 1
+        assert actual.iloc[0] == expected