|
| 1 | +"""Currency parsing: the decisions `test_currency_locale.py` does not pin. |
| 2 | +
|
| 3 | +Mutation testing over :mod:`freshdata.semantic.experts` found that the currency |
| 4 | +parser still passes its suite when several of its locale decisions are inverted. |
| 5 | +The existing suite checks the *values* that well-formed amounts parse to; it |
| 6 | +does not check the boundaries of the grouping validator, the "repeated |
| 7 | +separator can only be grouping" rule, or the ``ambiguous`` flag that decides |
| 8 | +whether a cell is auto-repaired or routed to a human. Each group below pins one |
| 9 | +of those, and each fails if the corresponding decision is flipped. |
| 10 | +
|
| 11 | +What is at stake in each group: |
| 12 | +
|
| 13 | +* **Grouping boundaries** -- ``_valid_grouping`` is the only thing standing |
| 14 | + between ``$1.2.3`` and a silent reading of 123. If its first-group length |
| 15 | + check is off by one, ordinary ``$123,456.78`` stops parsing; if its |
| 16 | + reject-branches are inverted, malformed money is coerced to a plausible |
| 17 | + number instead of being left alone. Both directions are pinned here. |
| 18 | +* **Digit gate** -- ``_split_amount`` refuses a body with no digits *before* |
| 19 | + handing it to ``float()``. Without that gate ``float()`` happily accepts |
| 20 | + ``"inf"`` and ``"nan"``, so a junk cell would become an infinite monetary |
| 21 | + amount rather than ``None``. |
| 22 | +* **The ambiguity flag** -- ``"1,000"`` with no currency to appeal to is either |
| 23 | + one thousand or 1.0 with a decimal comma, and nothing in the string settles |
| 24 | + it. ``_split_amount`` returns ``ambiguous=True`` so the caller can route the |
| 25 | + cell to review. If that flag is flipped to ``False`` the value is silently |
| 26 | + auto-applied at high confidence, which is exactly the thousand-fold error |
| 27 | + FD2-002 was about -- only this time with no signal that a guess was made. |
| 28 | + Nothing else in the suite asserts on the flag's ``True`` case. |
| 29 | +
|
| 30 | +These call the private ``_split_amount`` / ``_valid_grouping`` deliberately: |
| 31 | +``_split_amount``'s no-currency branch is not reachable through |
| 32 | +``parse_currency_parts`` (see the last test), so testing it at the public API |
| 33 | +alone cannot pin it. |
| 34 | +""" |
| 35 | + |
| 36 | +from __future__ import annotations |
| 37 | + |
| 38 | +import pytest |
| 39 | + |
| 40 | +from freshdata.semantic.experts import ( |
| 41 | + _split_amount, |
| 42 | + _valid_grouping, |
| 43 | + parse_currency, |
| 44 | + parse_currency_parts, |
| 45 | +) |
| 46 | + |
| 47 | +# -- the ambiguity flag ------------------------------------------------------ |
| 48 | + |
| 49 | + |
| 50 | +@pytest.mark.parametrize( |
| 51 | + ("body", "expected"), |
| 52 | + [ |
| 53 | + # One thousand under US convention, 1.0 under European convention. |
| 54 | + ("1,000", (1000.0, True)), |
| 55 | + ("1.000", (1.0, True)), |
| 56 | + ("1,200", (1200.0, True)), |
| 57 | + ("1.200", (1.2, True)), |
| 58 | + ], |
| 59 | +) |
| 60 | +def test_a_grouped_amount_without_a_currency_code_is_flagged_ambiguous(body, expected): |
| 61 | + """No currency and a 3-digit tail: the reading is a guess, and says so. |
| 62 | +
|
| 63 | + The value returned is the dot-as-decimal reading, but the second element |
| 64 | + must stay ``True`` so the caller routes the cell to a human instead of |
| 65 | + auto-applying a reading that is a coin flip. |
| 66 | + """ |
| 67 | + assert _split_amount(body, None) == expected |
| 68 | + |
| 69 | + |
| 70 | +def test_a_currency_code_removes_the_ambiguity_flag(): |
| 71 | + """Once a currency is known the reading is decided, not guessed.""" |
| 72 | + assert _split_amount("1,000", "USD") == (1000.0, False) |
| 73 | + assert _split_amount("1.000", "USD") == (1.0, False) |
| 74 | + # CHF writes the decimal comma, so its readings are the mirror image. |
| 75 | + assert _split_amount("1,000", "CHF") == (1.0, False) |
| 76 | + assert _split_amount("1.000", "CHF") == (1000.0, False) |
| 77 | + |
| 78 | + |
| 79 | +def test_a_known_currency_reads_a_three_digit_tail_by_its_own_convention(): |
| 80 | + """``$1.000`` is one dollar, not a thousand: USD writes the decimal dot.""" |
| 81 | + assert parse_currency_parts("$1.000") == (1.0, False) |
| 82 | + assert parse_currency_parts("$1.250") == (1.25, False) |
| 83 | + assert parse_currency_parts("EUR 1.000") == (1000.0, False) |
| 84 | + |
| 85 | + |
| 86 | +# -- grouping boundaries ----------------------------------------------------- |
| 87 | + |
| 88 | + |
| 89 | +@pytest.mark.parametrize( |
| 90 | + ("part", "sep"), |
| 91 | + [("123,456", ","), ("999,999,999", ","), ("123.456", ".")], |
| 92 | +) |
| 93 | +def test_a_three_digit_first_group_is_still_valid_grouping(part, sep): |
| 94 | + """The first group may be *up to* three digits, and three is allowed.""" |
| 95 | + assert _valid_grouping(part, sep) is True |
| 96 | + |
| 97 | + |
| 98 | +def test_an_amount_whose_first_group_is_exactly_three_digits_parses(): |
| 99 | + """The commonest shape of all -- rejecting it would break ordinary money.""" |
| 100 | + assert parse_currency("$123,456.78") == 123456.78 |
| 101 | + assert parse_currency("€123.456,78") == 123456.78 |
| 102 | + |
| 103 | + |
| 104 | +@pytest.mark.parametrize("part", ["1234,567", "12345,678"]) |
| 105 | +def test_a_first_group_longer_than_three_digits_is_not_grouping(part): |
| 106 | + assert _valid_grouping(part, ",") is False |
| 107 | + |
| 108 | + |
| 109 | +def test_an_over_long_first_group_is_rejected_rather_than_coerced(): |
| 110 | + """``$1234,567.00`` is not 1234567.00 under either convention. |
| 111 | +
|
| 112 | + Accepting it would mean inventing a grouping that the writer did not use, |
| 113 | + so the parser must decline and leave the cell to ordinary dtype repair. |
| 114 | + """ |
| 115 | + assert parse_currency("$1234,567.00") is None |
| 116 | + |
| 117 | + |
| 118 | +def test_a_thousands_group_containing_a_non_digit_is_rejected(): |
| 119 | + """``float()`` accepts underscores between digits; grouping must not. |
| 120 | +
|
| 121 | + ``float("1_00.25")`` is 100.25, so without the ``isdigit`` check on every |
| 122 | + group after the first, ``$1,_00.25`` would parse to 100.25 instead of being |
| 123 | + reported unparseable. |
| 124 | + """ |
| 125 | + assert _valid_grouping("1,_00", ",") is False |
| 126 | + assert parse_currency("$1,_00.25") is None |
| 127 | + |
| 128 | + |
| 129 | +# -- a repeated separator can only be grouping ------------------------------- |
| 130 | + |
| 131 | + |
| 132 | +@pytest.mark.parametrize( |
| 133 | + ("text", "expected"), |
| 134 | + [ |
| 135 | + ("$1.234.567", 1234567.0), |
| 136 | + ("$1.234.567,89", 1234567.89), |
| 137 | + ("€1,234,567", 1234567.0), |
| 138 | + ], |
| 139 | +) |
| 140 | +def test_a_separator_used_more_than_once_is_read_as_grouping(text, expected): |
| 141 | + """A decimal separator appears at most once, so twice means grouping. |
| 142 | +
|
| 143 | + This holds even when the separator is the one the currency normally uses as |
| 144 | + a decimal point: ``$1.234.567`` is a million, not a malformed 1.234. |
| 145 | + """ |
| 146 | + assert parse_currency(text) == expected |
| 147 | + |
| 148 | + |
| 149 | +# -- the digit gate before float() ------------------------------------------- |
| 150 | + |
| 151 | + |
| 152 | +@pytest.mark.parametrize("body", ["", "inf", "nan", "-inf", "infinity", ".", ",", "-"]) |
| 153 | +def test_a_body_with_no_digits_is_rejected_before_float_sees_it(body): |
| 154 | + """``float("inf")`` succeeds; an amount column must not inherit that. |
| 155 | +
|
| 156 | + Every one of these must come back ``(None, False)`` -- no value, and no |
| 157 | + claim that an ambiguity was resolved. |
| 158 | + """ |
| 159 | + assert _split_amount(body, "USD") == (None, False) |
| 160 | + assert _split_amount(body, None) == (None, False) |
| 161 | + |
| 162 | + |
| 163 | +def test_a_marker_with_no_number_reports_no_value_and_no_ambiguity(): |
| 164 | + assert parse_currency_parts("$") == (None, False) |
| 165 | + assert parse_currency_parts("EUR") == (None, False) |
| 166 | + |
| 167 | + |
| 168 | +def test_malformed_grouping_reports_no_value_and_no_ambiguity(): |
| 169 | + """Rejection is not ambiguity: there is no reading to route for review.""" |
| 170 | + assert parse_currency_parts("$1.2.3") == (None, False) |
| 171 | + assert parse_currency_parts("$1,20.50") == (None, False) |
| 172 | + assert parse_currency_parts("$1234,567.00") == (None, False) |
| 173 | + |
| 174 | + |
| 175 | +def test_a_string_without_a_currency_marker_reports_no_ambiguity(): |
| 176 | + """Not currency at all, so there is nothing for a human to adjudicate.""" |
| 177 | + assert parse_currency_parts("1,200") == (None, False) |
| 178 | + assert parse_currency_parts("1.200") == (None, False) |
| 179 | + |
| 180 | + |
| 181 | +# -- documented, not endorsed ------------------------------------------------ |
| 182 | + |
| 183 | + |
| 184 | +@pytest.mark.parametrize( |
| 185 | + ("text", "expected"), |
| 186 | + [("CAD 1.200", 1.2), ("AUD 1.200", 1.2), ("CNY 1.200", 1.2), ("$1.200", 1.2)], |
| 187 | +) |
| 188 | +def test_an_ambiguous_amount_reaching_the_public_parser_is_never_flagged(text, expected): |
| 189 | + """Current behaviour: ``ambiguous=True`` cannot surface through the API. |
| 190 | +
|
| 191 | + ``parse_currency_parts`` requires a currency marker, and every marker it |
| 192 | + accepts is one ``detect_currency`` also resolves, so ``_split_amount`` is |
| 193 | + never called with ``code=None`` from here. The "report it ambiguous rather |
| 194 | + than guess" branch is therefore unreachable in production, and a currency |
| 195 | + outside ``_COMMA_DECIMAL_CURRENCIES`` is guessed as dot-decimal instead -- |
| 196 | + contrary to what the module comment on that table says. Pinned as the |
| 197 | + behaviour that exists today; see the note filed with this suite. |
| 198 | + """ |
| 199 | + assert parse_currency_parts(text) == (expected, False) |
0 commit comments