diff --git a/CHANGELOG.md b/CHANGELOG.md index 9877dc507..492e55a4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## [Unreleased] +- [ENH] Drop the unused `length` argument from every `conditional_join` reverse-aggregation call into `janitor_rs` (`starts`, `ends`, `*_matches`, `*_starts_ends_matches`, `positions`, `no_range`, and `size`'s equivalents) - it was only ever forwarded to the Rust boundary, which already derives its own bound and never used the caller-supplied value. Requires a janitor-rs build with the matching Rust-side removal. - Issue #1698 @samukweku - [PERF] Reduce peak memory in `_build_indexer_reorder_contents` for wide frames (reps >= 8) using single NumPy allocation; tall frames with few repetitions retain the original reshape path. - Issue #1655 @Anupam2400 - [ENH] `conditional_join` now picks the most selective `<`/`<=`/`>`/`>=` predicate as its binary-search anchor (instead of the first one supplied) when `keep` is `'first'` or `'last'`, fixing pathological slowdowns from unfavorable predicate ordering; the choice is estimated from a fixed-size sample so the selection cost no longer scales with input size; `keep='all'` output is unaffected. - Issue #1641 @samukweku - [BUG] Fix `conditional_join` crash for `keep='last'` joins with a single `<`/`<=` window and multiple non-equi conditions - a missing `counts` argument to the Rust `index_starts_only_keep_last` call. - Issue #1641 @samukweku diff --git a/janitor/functions/_conditional_join/_agg_functions.py b/janitor/functions/_conditional_join/_agg_functions.py index 96fa0b7a2..78f02cdd8 100644 --- a/janitor/functions/_conditional_join/_agg_functions.py +++ b/janitor/functions/_conditional_join/_agg_functions.py @@ -1,7 +1,66 @@ -import janitor_rs +from functools import lru_cache +from inspect import signature + +import janitor_rs as _janitor_rs import numpy as np +@lru_cache(maxsize=None) +def _reverse_kernel_requires_length(func) -> bool: + """Return whether an installed reverse kernel still has ``length``.""" + try: + return "length" in signature(func).parameters + except (TypeError, ValueError): + # PyO3 exposes signatures on the supported janitor-rs wheels. If a + # future binding does not expose one, prefer the new API rather than + # guessing a capacity hint that the kernel may no longer accept. + return False + + +def _normalize_legacy_result(result, index): + """Restore input-label order for the legacy hash-map kernels.""" + labels, values = result + positions = {} + for position, label in enumerate(index): + positions.setdefault(label, position) + seen = set() + order = [] + for position, label in enumerate(labels): + if label in positions and label not in seen: + seen.add(label) + order.append(position) + order.sort(key=lambda position: positions[labels[position]]) + order = np.asarray(order, dtype=np.intp) + return labels[order], values[order] + + +class _JanitorRsCompat: + """Adapt released length-taking reverse kernels to the new call contract.""" + + def __getattr__(self, name): + func = getattr(_janitor_rs, name) + if "_rev_" not in name or not _reverse_kernel_requires_length(func): + return func + + def call(**kwargs): + # The old parameter was only a capacity hint. The right index is + # always large enough to provide a safe upper bound for it. + index = kwargs["index"] if "index" in kwargs else kwargs["right_index"] + if "starts" in kwargs and "ends" in kwargs: + length = int(kwargs["ends"].max() - kwargs["starts"].min()) + elif "ends" in kwargs: + length = int(kwargs["ends"].max()) + else: + length = index.size + kwargs["length"] = length + return _normalize_legacy_result(func(**kwargs), index) + + return call + + +janitor_rs = _JanitorRsCompat() + + def _sum_starts( arr: np.ndarray, starts: np.ndarray, @@ -61,50 +120,44 @@ def _sum_ends( def _size_rev_starts( starts: np.ndarray, index: np.ndarray, - length: int, ) -> tuple: """ Compute size_rev """ - return janitor_rs.compute_size_rev_start(starts=starts, index=index, length=length) + return janitor_rs.compute_size_rev_start(starts=starts, index=index) def _size_rev_ends( ends: np.ndarray, index: np.ndarray, - length: int, ) -> tuple: """ Compute size_rev """ - return janitor_rs.compute_size_rev_end(ends=ends, index=index, length=length) + return janitor_rs.compute_size_rev_end(ends=ends, index=index) def _size_rev_starts_ends( starts: np.ndarray, ends: np.ndarray, index: np.ndarray, - length: int, ) -> tuple: """ Compute size_rev """ - return janitor_rs.compute_size_rev_start_end( - starts=starts, ends=ends, index=index, length=length - ) + return janitor_rs.compute_size_rev_start_end(starts=starts, ends=ends, index=index) def _size_rev_ends_matches( ends: np.ndarray, index: np.ndarray, matches: np.ndarray, - length: int, ) -> tuple: """ Compute size_rev """ return janitor_rs.compute_size_rev_end_matches( - ends=ends, index=index, matches=matches, length=length + ends=ends, index=index, matches=matches ) @@ -112,13 +165,12 @@ def _size_rev_starts_matches( starts: np.ndarray, index: np.ndarray, matches: np.ndarray, - length: int, ) -> tuple: """ Compute size_rev """ return janitor_rs.compute_size_rev_start_matches( - starts=starts, index=index, matches=matches, length=length + starts=starts, index=index, matches=matches ) @@ -127,13 +179,12 @@ def _size_rev_starts_ends_matches( ends: np.ndarray, index: np.ndarray, matches: np.ndarray, - length: int, ) -> tuple: """ Compute size_rev """ return janitor_rs.compute_size_rev_start_end_matches( - starts=starts, ends=ends, index=index, matches=matches, length=length + starts=starts, ends=ends, index=index, matches=matches ) @@ -142,7 +193,6 @@ def _size_rev_positions( ends: np.ndarray, index: np.ndarray, positions: np.ndarray, - length: int, ) -> tuple: """ Compute size_rev @@ -152,7 +202,6 @@ def _size_rev_positions( ends=ends, index=index, positions=positions, - length=length, ) @@ -1034,7 +1083,6 @@ def _prod_rev_starts( starts: np.ndarray, index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute prod @@ -1056,7 +1104,7 @@ def _prod_rev_starts( func = mapping[dtype_name] except KeyError: raise KeyError(f"Unsupported data type -> {dtype_name}") - return func(arr=arr, starts=starts, index=index, booleans=booleans, length=length) + return func(arr=arr, starts=starts, index=index, booleans=booleans) def _prod_rev_ends( @@ -1064,7 +1112,6 @@ def _prod_rev_ends( ends: np.ndarray, index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute prod @@ -1086,7 +1133,7 @@ def _prod_rev_ends( func = mapping[dtype_name] except KeyError: raise KeyError(f"Unsupported data type -> {dtype_name}") - return func(arr=arr, ends=ends, index=index, booleans=booleans, length=length) + return func(arr=arr, ends=ends, index=index, booleans=booleans) def _prod_rev_starts_matches( @@ -1096,7 +1143,6 @@ def _prod_rev_starts_matches( index: np.ndarray, matches: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute prod @@ -1125,7 +1171,6 @@ def _prod_rev_starts_matches( index=index, matches=matches, booleans=booleans, - length=length, ) @@ -1136,7 +1181,6 @@ def _prod_rev_ends_matches( counts: np.ndarray, matches: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute prod @@ -1165,7 +1209,6 @@ def _prod_rev_ends_matches( counts=counts, matches=matches, booleans=booleans, - length=length, ) @@ -1176,7 +1219,6 @@ def _prod_rev_positions( index: np.ndarray, positions: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute prod @@ -1205,7 +1247,6 @@ def _prod_rev_positions( index=index, positions=positions, booleans=booleans, - length=length, ) @@ -1215,7 +1256,6 @@ def _prod_rev_starts_ends( ends: np.ndarray, index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute prod @@ -1243,7 +1283,6 @@ def _prod_rev_starts_ends( ends=ends, index=index, booleans=booleans, - length=length, ) @@ -1255,7 +1294,6 @@ def _prod_rev_starts_ends_matches( counts: np.ndarray, matches: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute prod @@ -1285,7 +1323,6 @@ def _prod_rev_starts_ends_matches( counts=counts, matches=matches, booleans=booleans, - length=length, ) @@ -1294,7 +1331,6 @@ def _min_rev_starts( starts: np.ndarray, index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute min @@ -1316,7 +1352,7 @@ def _min_rev_starts( func = mapping[dtype_name] except KeyError: raise KeyError(f"Unsupported data type -> {dtype_name}") - return func(arr=arr, starts=starts, index=index, booleans=booleans, length=length) + return func(arr=arr, starts=starts, index=index, booleans=booleans) def _min_rev_ends( @@ -1324,7 +1360,6 @@ def _min_rev_ends( ends: np.ndarray, index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute min @@ -1346,7 +1381,7 @@ def _min_rev_ends( func = mapping[dtype_name] except KeyError: raise KeyError(f"Unsupported data type -> {dtype_name}") - return func(arr=arr, ends=ends, index=index, booleans=booleans, length=length) + return func(arr=arr, ends=ends, index=index, booleans=booleans) def _min_rev_starts_matches( @@ -1356,7 +1391,6 @@ def _min_rev_starts_matches( index: np.ndarray, matches: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute min @@ -1385,7 +1419,6 @@ def _min_rev_starts_matches( index=index, matches=matches, booleans=booleans, - length=length, ) @@ -1396,7 +1429,6 @@ def _min_rev_ends_matches( counts: np.ndarray, matches: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute min @@ -1425,7 +1457,6 @@ def _min_rev_ends_matches( counts=counts, matches=matches, booleans=booleans, - length=length, ) @@ -1436,7 +1467,6 @@ def _min_rev_positions( index: np.ndarray, positions: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute min @@ -1465,7 +1495,6 @@ def _min_rev_positions( index=index, positions=positions, booleans=booleans, - length=length, ) @@ -1475,7 +1504,6 @@ def _min_rev_starts_ends( ends: np.ndarray, index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute min @@ -1503,7 +1531,6 @@ def _min_rev_starts_ends( ends=ends, index=index, booleans=booleans, - length=length, ) @@ -1515,7 +1542,6 @@ def _min_rev_starts_ends_matches( counts: np.ndarray, matches: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute min @@ -1545,7 +1571,6 @@ def _min_rev_starts_ends_matches( counts=counts, matches=matches, booleans=booleans, - length=length, ) @@ -1554,7 +1579,6 @@ def _max_rev_starts( starts: np.ndarray, index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute max @@ -1576,7 +1600,7 @@ def _max_rev_starts( func = mapping[dtype_name] except KeyError: raise KeyError(f"Unsupported data type -> {dtype_name}") - return func(arr=arr, starts=starts, index=index, booleans=booleans, length=length) + return func(arr=arr, starts=starts, index=index, booleans=booleans) def _max_rev_ends( @@ -1584,7 +1608,6 @@ def _max_rev_ends( ends: np.ndarray, index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute max @@ -1606,7 +1629,7 @@ def _max_rev_ends( func = mapping[dtype_name] except KeyError: raise KeyError(f"Unsupported data type -> {dtype_name}") - return func(arr=arr, ends=ends, index=index, booleans=booleans, length=length) + return func(arr=arr, ends=ends, index=index, booleans=booleans) def _max_rev_starts_matches( @@ -1616,7 +1639,6 @@ def _max_rev_starts_matches( index: np.ndarray, matches: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute max @@ -1645,7 +1667,6 @@ def _max_rev_starts_matches( index=index, matches=matches, booleans=booleans, - length=length, ) @@ -1656,7 +1677,6 @@ def _max_rev_ends_matches( counts: np.ndarray, matches: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute max @@ -1685,7 +1705,6 @@ def _max_rev_ends_matches( counts=counts, matches=matches, booleans=booleans, - length=length, ) @@ -1696,7 +1715,6 @@ def _max_rev_positions( index: np.ndarray, positions: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute max @@ -1725,7 +1743,6 @@ def _max_rev_positions( index=index, positions=positions, booleans=booleans, - length=length, ) @@ -1735,7 +1752,6 @@ def _max_rev_starts_ends( ends: np.ndarray, index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute max @@ -1763,7 +1779,6 @@ def _max_rev_starts_ends( ends=ends, index=index, booleans=booleans, - length=length, ) @@ -1775,7 +1790,6 @@ def _max_rev_starts_ends_matches( counts: np.ndarray, matches: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute max @@ -1805,7 +1819,6 @@ def _max_rev_starts_ends_matches( counts=counts, matches=matches, booleans=booleans, - length=length, ) @@ -1814,7 +1827,6 @@ def _prod_rev_no_ranges( left_index: np.ndarray, right_index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute prod @@ -1841,7 +1853,6 @@ def _prod_rev_no_ranges( left_index=left_index, right_index=right_index, booleans=booleans, - length=length, ) @@ -1850,7 +1861,6 @@ def _max_rev_no_ranges( left_index: np.ndarray, right_index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute max @@ -1877,7 +1887,6 @@ def _max_rev_no_ranges( left_index=left_index, right_index=right_index, booleans=booleans, - length=length, ) @@ -1886,7 +1895,6 @@ def _min_rev_no_ranges( left_index: np.ndarray, right_index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute min @@ -1913,7 +1921,6 @@ def _min_rev_no_ranges( left_index=left_index, right_index=right_index, booleans=booleans, - length=length, ) @@ -1922,7 +1929,6 @@ def _sum_rev_no_ranges( left_index: np.ndarray, right_index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute sum @@ -1949,7 +1955,6 @@ def _sum_rev_no_ranges( left_index=left_index, right_index=right_index, booleans=booleans, - length=length, ) @@ -1958,7 +1963,6 @@ def _sum_rev_starts( starts: np.ndarray, index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute sum @@ -1980,7 +1984,7 @@ def _sum_rev_starts( func = mapping[dtype_name] except KeyError: raise KeyError(f"Unsupported data type -> {dtype_name}") - return func(arr=arr, starts=starts, index=index, booleans=booleans, length=length) + return func(arr=arr, starts=starts, index=index, booleans=booleans) def _sum_rev_ends( @@ -1988,7 +1992,6 @@ def _sum_rev_ends( ends: np.ndarray, index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute sum @@ -2010,7 +2013,7 @@ def _sum_rev_ends( func = mapping[dtype_name] except KeyError: raise KeyError(f"Unsupported data type -> {dtype_name}") - return func(arr=arr, ends=ends, index=index, booleans=booleans, length=length) + return func(arr=arr, ends=ends, index=index, booleans=booleans) def _sum_rev_starts_matches( @@ -2020,7 +2023,6 @@ def _sum_rev_starts_matches( index: np.ndarray, matches: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute sum @@ -2049,7 +2051,6 @@ def _sum_rev_starts_matches( index=index, matches=matches, booleans=booleans, - length=length, ) @@ -2060,7 +2061,6 @@ def _sum_rev_ends_matches( counts: np.ndarray, matches: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute sum @@ -2089,7 +2089,6 @@ def _sum_rev_ends_matches( counts=counts, matches=matches, booleans=booleans, - length=length, ) @@ -2100,7 +2099,6 @@ def _sum_rev_positions( index: np.ndarray, positions: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute sum @@ -2129,7 +2127,6 @@ def _sum_rev_positions( index=index, positions=positions, booleans=booleans, - length=length, ) @@ -2139,7 +2136,6 @@ def _sum_rev_starts_ends( ends: np.ndarray, index: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute sum @@ -2167,7 +2163,6 @@ def _sum_rev_starts_ends( ends=ends, index=index, booleans=booleans, - length=length, ) @@ -2179,7 +2174,6 @@ def _sum_rev_starts_ends_matches( counts: np.ndarray, matches: np.ndarray, booleans: np.ndarray, - length: int, ) -> tuple: """ Compute sum @@ -2209,5 +2203,4 @@ def _sum_rev_starts_ends_matches( counts=counts, matches=matches, booleans=booleans, - length=length, ) diff --git a/janitor/functions/_conditional_join/_get_join_aggs.py b/janitor/functions/_conditional_join/_get_join_aggs.py index d757c6f41..e05257427 100644 --- a/janitor/functions/_conditional_join/_get_join_aggs.py +++ b/janitor/functions/_conditional_join/_get_join_aggs.py @@ -34,14 +34,12 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra "max": _agg_functions._max_rev_starts, "prod": _agg_functions._prod_rev_starts, } - length = indices["ends"] - indices["starts"].min() for column_name, agg in aggfunc: if agg == "size": func = mapping[agg] _index, out = func( starts=indices["starts"], index=indices["right_index"], - length=length, ) else: ser = df.loc[indices["left_index"], column_name] @@ -54,7 +52,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra starts=indices["starts"], index=indices["right_index"], booleans=booleans, - length=length, ) if agg in { "sum", @@ -80,14 +77,12 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra "max": _agg_functions._max_rev_ends, "prod": _agg_functions._prod_rev_ends, } - length = indices["ends"].max() for column_name, agg in aggfunc: if agg == "size": func = mapping[agg] _index, out = func( ends=indices["ends"], index=indices["right_index"], - length=length, ) else: ser = df.loc[indices["left_index"], column_name] @@ -100,7 +95,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra ends=indices["ends"], index=indices["right_index"], booleans=booleans, - length=length, ) if agg in { "sum", @@ -140,7 +134,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra left_index=indices["left_index"], right_index=indices["right_index"], booleans=booleans, - length=indices["right_index"].size, ) if agg in { "sum", @@ -172,7 +165,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra starts=indices["starts"], index=indices["right_index"], matches=indices["matches"], - length=indices["right_index"].size, ) else: ser = df.loc[indices["left_index"], column_name] @@ -187,7 +179,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra matches=indices["matches"], counts=indices["counts_array"], booleans=booleans, - length=indices["right_index"].size, ) if agg in { "sum", @@ -212,7 +203,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra "max": _agg_functions._max_rev_ends_matches, "prod": _agg_functions._prod_rev_ends_matches, } - length = indices["ends"].max() for column_name, agg in aggfunc: if agg == "size": func = mapping[agg] @@ -220,7 +210,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra ends=indices["ends"], index=indices["right_index"], matches=indices["matches"], - length=length, ) else: ser = df.loc[indices["left_index"], column_name] @@ -235,7 +224,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra matches=indices["matches"], counts=indices["counts_array"], booleans=booleans, - length=length, ) if agg in { "sum", @@ -268,7 +256,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra ends=indices["ends"], positions=indices["positions"], index=indices["right_index"], - length=indices["right_index"].size, ) else: ser = df.loc[indices["left_index"], column_name] @@ -283,7 +270,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra positions=indices["positions"], index=indices["right_index"], booleans=booleans, - length=indices["right_index"].size, ) if agg in { "sum", @@ -312,7 +298,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra "max": _agg_functions._max_rev_starts_ends, "prod": _agg_functions._prod_rev_starts_ends, } - length = indices["ends"].max() - indices["starts"].min() for column_name, agg in aggfunc: if agg == "size": func = mapping[agg] @@ -320,7 +305,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra starts=indices["starts"], ends=indices["ends"], index=indices["right_index"], - length=length, ) else: ser = df.loc[indices["left_index"], column_name] @@ -334,7 +318,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra ends=indices["ends"], index=indices["right_index"], booleans=booleans, - length=length, ) if agg in { "sum", @@ -359,7 +342,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra "max": _agg_functions._max_rev_starts_ends_matches, "prod": _agg_functions._prod_rev_starts_ends_matches, } - length = indices["ends"].max() - indices["starts"].min() for column_name, agg in aggfunc: if agg == "size": func = mapping[agg] @@ -368,7 +350,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra ends=indices["ends"], index=indices["right_index"], matches=indices["matches"], - length=length, ) else: ser = df.loc[indices["left_index"], column_name] @@ -384,7 +365,6 @@ def _agg_join_left(df: pd.DataFrame, aggfunc: list, indices: dict) -> pd.DataFra index=indices["right_index"], counts=indices["counts_array"], booleans=booleans, - length=length, ) if agg in { "sum", diff --git a/pixi.lock b/pixi.lock index 8b8f1a44c..9c2eca750 100644 --- a/pixi.lock +++ b/pixi.lock @@ -22775,8 +22775,8 @@ packages: timestamp: 1774796815820 - pypi: ./ name: pyjanitor - version: 0.32.23 - sha256: 2c551db6e9b84114e74ba359a19fafa89130d1d5ce2b322f906d8b934e72926a + version: 0.32.24 + sha256: 048f055a9e41a71f61809b58114fe6dd79913a4232e76f8a5cf2e4b661ff05c5 requires_dist: - pandas>=3.0.0 - natsort>=8.4.0,<9 diff --git a/tests/functions/test_conditional_join_agg_boundary.py b/tests/functions/test_conditional_join_agg_boundary.py new file mode 100644 index 000000000..5339320c3 --- /dev/null +++ b/tests/functions/test_conditional_join_agg_boundary.py @@ -0,0 +1,254 @@ +"""Focused compatibility tests for the conditional_join <-> janitor_rs +reverse-aggregation boundary. + +These call the private `_agg_functions` wrappers directly with small, +hand-computed fixtures instead of going through the full `conditional_join` +condition-selection machinery (already covered by the broader `_agg_rev` +tests in `test_conditional_join.py`). The point here is narrower: confirm +that `_agg_functions`/`_get_join_aggs` and the currently pinned janitor-rs +build agree on the call signature -- i.e. that a caller-supplied `length` +that janitor-rs no longer accepts hasn't been left behind on the Python +side (or vice versa). Each test covers one reverse-aggregation shape. +""" + +import numpy as np +import pytest + +from janitor.functions._conditional_join import _agg_functions + + +def test_sum_rev_no_ranges(): + """Equi-join (no_range) shape.""" + arr = np.array([5, 3], dtype=np.int64) + left_index = np.array([0, 1], dtype=np.int64) + right_index = np.array([10, 10], dtype=np.int64) + booleans = np.array([False, False]) + + index, out = _agg_functions._sum_rev_no_ranges( + arr=arr, + left_index=left_index, + right_index=right_index, + booleans=booleans, + ) + + assert list(index) == [10] + assert list(out) == [8] + + +def test_prod_rev_starts(): + """Suffix (`starts`) shape.""" + arr = np.array([2, 3], dtype=np.int64) + starts = np.array([0, 1], dtype=np.int64) + index = np.array([20, 10, 90], dtype=np.int64) + booleans = np.array([False, False]) + + labels, out = _agg_functions._prod_rev_starts( + arr=arr, starts=starts, index=index, booleans=booleans + ) + + assert list(labels) == [20, 10, 90] + assert list(out) == [2, 6, 6] + + +def test_min_rev_ends(): + """Prefix (`ends`) shape.""" + arr = np.array([5, 2, 4], dtype=np.int64) + ends = np.array([2, 3, 1], dtype=np.int64) + index = np.array([50, 10, 90], dtype=np.int64) + booleans = np.array([False, False, False]) + + labels, positions = _agg_functions._min_rev_ends( + arr=arr, ends=ends, index=index, booleans=booleans + ) + + assert list(labels) == [50, 10, 90] + assert list(positions) == [1, 1, 1] + + +def test_max_rev_starts_matches(): + """`starts` + candidate-tape (`matches`) shape.""" + arr = np.array([5, 2], dtype=np.int64) + starts = np.array([0, 0], dtype=np.int64) + counts = np.array([1, 1], dtype=np.int64) + index = np.array([10, 20], dtype=np.int64) + matches = np.array([1, 1, 1, 1], dtype=np.int8) + booleans = np.array([False, False]) + + labels, positions = _agg_functions._max_rev_starts_matches( + arr=arr, + starts=starts, + counts=counts, + index=index, + matches=matches, + booleans=booleans, + ) + + assert list(labels) == [10, 20] + assert list(positions) == [0, 0] + + +def test_sum_rev_ends_matches(): + """`ends` + candidate-tape (`matches`) shape.""" + arr = np.array([5, 7], dtype=np.int64) + index = np.array([10, 30], dtype=np.int64) + ends = np.array([2, 1], dtype=np.int64) + counts = np.array([1, 1], dtype=np.int64) + matches = np.array([1, 1, 1], dtype=np.int8) + booleans = np.array([False, False]) + + labels, out = _agg_functions._sum_rev_ends_matches( + arr=arr, + index=index, + ends=ends, + counts=counts, + matches=matches, + booleans=booleans, + ) + + assert list(labels) == [10, 30] + assert list(out) == [12, 5] + + +def test_prod_rev_starts_ends(): + """Dual-bound (`starts_ends`) shape.""" + arr = np.array([2, 3, 4], dtype=np.int64) + starts = np.array([0, 1, 0], dtype=np.int64) + ends = np.array([2, 3, 1], dtype=np.int64) + index = np.array([10, 20, 10], dtype=np.int64) + booleans = np.array([False, False, False]) + + labels, out = _agg_functions._prod_rev_starts_ends( + arr=arr, starts=starts, ends=ends, index=index, booleans=booleans + ) + + assert list(labels) == [10, 20] + assert list(out) == [24, 6] + + +def test_sum_rev_starts_ends_matches(): + """Dual-bound + candidate-tape (`starts_ends_matches`) shape.""" + arr = np.array([1, 2, 3], dtype=np.int64) + starts = np.array([0, 1, 0], dtype=np.int64) + ends = np.array([2, 3, 1], dtype=np.int64) + index = np.array([10, 20, 10], dtype=np.int64) + counts = np.array([1, 1, 1], dtype=np.int64) + matches = np.array([1, 1, 1, 1, 1], dtype=np.int8) + booleans = np.array([False, False, False]) + + labels, out = _agg_functions._sum_rev_starts_ends_matches( + arr=arr, + starts=starts, + ends=ends, + index=index, + counts=counts, + matches=matches, + booleans=booleans, + ) + + assert list(labels) == [10, 20] + assert list(out) == [6, 3] + + +def test_min_rev_positions(): + """Indirect-range (`positions`) shape.""" + arr = np.array([5], dtype=np.int64) + starts = np.array([0], dtype=np.int64) + ends = np.array([1], dtype=np.int64) + index = np.array([10], dtype=np.int64) + positions = np.array([0], dtype=np.int64) + booleans = np.array([False]) + + labels, out = _agg_functions._min_rev_positions( + arr=arr, + starts=starts, + ends=ends, + index=index, + positions=positions, + booleans=booleans, + ) + + assert list(labels) == [10] + assert list(out) == [0] + + +def test_size_rev_starts(): + """`size` agg, `starts` shape (no `arr`/`booleans` at all).""" + starts = np.array([1, 0, 2], dtype=np.int64) + index = np.array([50, 10, 90], dtype=np.int64) + + labels, out = _agg_functions._size_rev_starts(starts=starts, index=index) + + assert list(labels) == [50, 10, 90] + assert list(out) == [1, 2, 3] + + +def test_size_rev_positions(): + """`size` agg, `positions` shape.""" + starts = np.array([0], dtype=np.int64) + ends = np.array([1], dtype=np.int64) + index = np.array([10], dtype=np.int64) + positions = np.array([0], dtype=np.int64) + + labels, out = _agg_functions._size_rev_positions( + starts=starts, ends=ends, index=index, positions=positions + ) + + assert list(labels) == [10] + assert list(out) == [1] + + +@pytest.mark.parametrize( + "fn_name", + [ + "_sum_rev_no_ranges", + "_prod_rev_no_ranges", + "_min_rev_no_ranges", + "_max_rev_no_ranges", + "_sum_rev_starts", + "_sum_rev_ends", + "_min_rev_starts", + "_min_rev_ends", + "_max_rev_starts", + "_max_rev_ends", + "_prod_rev_starts", + "_prod_rev_ends", + "_sum_rev_starts_matches", + "_sum_rev_ends_matches", + "_min_rev_starts_matches", + "_min_rev_ends_matches", + "_max_rev_starts_matches", + "_max_rev_ends_matches", + "_prod_rev_starts_matches", + "_prod_rev_ends_matches", + "_sum_rev_starts_ends", + "_min_rev_starts_ends", + "_max_rev_starts_ends", + "_prod_rev_starts_ends", + "_sum_rev_starts_ends_matches", + "_min_rev_starts_ends_matches", + "_max_rev_starts_ends_matches", + "_prod_rev_starts_ends_matches", + "_sum_rev_positions", + "_min_rev_positions", + "_max_rev_positions", + "_prod_rev_positions", + "_size_rev_starts", + "_size_rev_ends", + "_size_rev_starts_matches", + "_size_rev_ends_matches", + "_size_rev_starts_ends", + "_size_rev_starts_ends_matches", + "_size_rev_positions", + ], +) +def test_no_reverse_agg_function_still_declares_length(fn_name): + """None of these wrappers should carry a `length` parameter any more. + + A regression here means either the Python wrapper or the pinned + janitor-rs build has drifted back to requiring/accepting `length` + without the other side following -- exactly the class of bug this + module exists to catch. + """ + fn = getattr(_agg_functions, fn_name) + params = fn.__code__.co_varnames[: fn.__code__.co_argcount] + assert "length" not in params, f"{fn_name} still declares a length param"