From b45b2eed027753e259fc627e3c097b92996d4228 Mon Sep 17 00:00:00 2001 From: samukweku Date: Wed, 19 Aug 2026 17:27:02 +1000 Subject: [PATCH 1/4] [ENH] add anti joins to conditional_join --- AGENTS.md | 21 +++- CHANGELOG.md | 2 + janitor/functions/conditional_join.py | 113 ++++++++++++++++++---- pixi.lock | 4 +- tests/functions/test_conditional_join.py | 118 +++++++++++++++++++++++ 5 files changed, 234 insertions(+), 24 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 34a7f1921..6c1a60f3b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,6 +37,9 @@ these into the main sections. task. - **Test-Driven**: Always run tests after making code changes. - **Document**: Keep docstrings up-to-date using Google-style format. +- **Explain Simply**: For complex public behavior, include a concise ELI5 + section in the docstring or nearby documentation. Explain the user-facing + mental model, not line-by-line implementation details. Skip trivial helpers. - **Lint Markdown**: Always run `markdownlint` on markdown files after editing. --- @@ -114,7 +117,7 @@ pixi run -e | Task | Command | |------|---------| -| Run all tests | `pixi run test` | +| Run all tests | `pixi run -e tests test` | | Run specific test | `pixi run pytest tests/functions/test_clean_names.py` | | Run tests matching pattern | `pixi run pytest -k "test_clean_names" -v` | | Run tests with coverage | `pixi run pytest --cov=janitor` | @@ -548,6 +551,22 @@ CLI. include MkDocs. The documentation task is available in the `docs` environment. **Recommendation**: Run `pixi run -e docs build-docs` to build documentation. +### [2026-08-19] Select the Tests Environment for the Full Suite + +**Context**: Running the full test suite with `pixi run test`. +**Learning**: The `test` task exists in multiple pixi environments, so the +unqualified command is ambiguous. +**Recommendation**: Run the full suite with `pixi run -e tests test`. + +### [2026-08-19] Add ELI5 Explanations for Complex Behavior + +**Context**: Documenting a feature with several join modes and edge cases. +**Learning**: Complex public behavior should include a short, plain-language +mental model in the code documentation as well as technical API details. +**Recommendation**: Add an ELI5 section to non-trivial public docstrings or +nearby documentation. Do not add redundant ELI5 comments to straightforward +helpers. + --- ## Version History diff --git a/CHANGELOG.md b/CHANGELOG.md index 549615ff2..d3eb7b1c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - [ENH] Replace `axis=1` `.apply()` with vectorized operations in `compare_df_cols` for 4-7x performance improvement. - Issue #1630 @Anupam2400 - [ENH] Replace `.apply()` with `.map()` in `find_replace` for ~6x performance improvement. - Issue #1422 @Anupam2400 - [ENH] Add `drop_first` parameter to `expand_column`. - Issue #368 @manav252 +- [ENH] Add `left_anti` and `right_anti` joins to `conditional_join`. - + Issue #1627 @samukweku - [ENH] Add `include_join_positions` parameter to `conditional_join`; added limited support for join aggregations via the `join_agg` function. - Issue #1497 @samukweku - [ENH] Added `rle_id` function for run-length encoding IDs - Issue #1435 @emmanuel-ferdman - [ENH] Add `scale_mad` for robust median/MAD scaling. - PR #1530 @ShachiMistry diff --git a/janitor/functions/conditional_join.py b/janitor/functions/conditional_join.py index faccbeb3c..9c3249583 100644 --- a/janitor/functions/conditional_join.py +++ b/janitor/functions/conditional_join.py @@ -43,7 +43,9 @@ def conditional_join( df: pd.DataFrame, right: pd.DataFrame | pd.Series, *conditions: tuple, - how: Literal["inner", "left", "right", "outer"] = "inner", + how: Literal[ + "inner", "left", "right", "outer", "left_anti", "right_anti" + ] = "inner", df_columns: Optional[Any] = slice(None), right_columns: Optional[Any] = slice(None), keep: Literal["first", "last", "all"] = "all", @@ -93,6 +95,14 @@ def conditional_join( For multiple conditions, the and(`&`) operator is used to combine the results of the individual conditions. + !!! info "ELI5" + + Imagine comparing every row on the left with rows on the right using + the rules you provide. An inner join keeps successful pairs. Left and + right joins also keep unmatched rows from their named side. A + `left_anti` join keeps only left rows that never found a successful + partner; `right_anti` does the same for right rows. + In some scenarios there might be performance gains if the less than join, or the greater than join condition, or the range condition is executed before the equi join - pass `force=True` to force this. @@ -105,7 +115,8 @@ def conditional_join( For non-equi joins, only numeric, timedelta and date columns are supported. - `inner`, `left`, `right` and `outer` joins are supported. + `inner`, `left`, `right`, `outer`, `left_anti` and `right_anti` + joins are supported. If the columns from `df` and `right` have nothing in common, a single index column is returned; else, a MultiIndex column @@ -243,6 +254,17 @@ def conditional_join( 9 NaN 12.0 15.0 right_only 10 NaN 0.0 1.0 right_only + Return rows from the left dataframe that have no match: + >>> df1.conditional_join( + ... df2, + ... ("value_1", "value_2A", ">"), + ... ("value_1", "value_2B", "<"), + ... how="left_anti", + ... ) + value_1 value_2A value_2B + 0 7 NaN NaN + 1 1 NaN NaN + !!! abstract "Version Changed" - 0.24.0 @@ -262,6 +284,8 @@ def conditional_join( - 0.32.10 - Added `include_join_positions` parameter. - Added `join_algorithm` parameter. + - 0.33.0 + - Added `left_anti` and `right_anti` joins. Args: df: A pandas DataFrame. @@ -275,7 +299,10 @@ def conditional_join( the and(`&`) operator is used to combine the results of the individual conditions. how: Indicates the type of join to be performed. - It can be one of `inner`, `left`, `right` or `outer`. + It can be one of `inner`, `left`, `right`, `outer`, + `left_anti` or `right_anti`. + `df_columns` cannot be `None` for a `left_anti` join, + and `right_columns` cannot be `None` for a `right_anti` join. df_columns: Columns to select from `df` in the final output dataframe. Column selection is based on the [`select_columns`][janitor.functions.select.select_columns] syntax. @@ -403,8 +430,16 @@ def _conditional_join_preliminary_checks( check("how", how, [str]) - if how not in {"inner", "left", "right", "outer"}: - raise ValueError("'how' should be one of 'inner', 'left', 'right' or 'outer'.") + join_types = {"inner", "left", "right", "outer", "left_anti", "right_anti"} + if how not in join_types: + raise ValueError( + "'how' should be one of 'inner', 'left', 'right', 'outer', " + "'left_anti' or 'right_anti'." + ) + if (how == "left_anti") and (df_columns is None): + raise ValueError("df_columns cannot be None for a left_anti join.") + if (how == "right_anti") and (right_columns is None): + raise ValueError("right_columns cannot be None for a right_anti join.") check("keep", keep, [str]) @@ -579,12 +614,13 @@ def _conditional_join_compute( le_lt_check = True df.index = range(len(df)) right.index = range(len(right)) + match_keep = "all" if how in {"left_anti", "right_anti"} else keep if eq_check: indices = _multiple_conditional_join_eq( df=df, right=right, conditions=conditions, - keep=keep, + keep=match_keep, use_numba=use_numba, force=force, return_matching_indices=return_building_blocks or aggfunc, @@ -595,7 +631,7 @@ def _conditional_join_compute( df=df, right=right, conditions=conditions, - keep=keep, + keep=match_keep, use_numba=use_numba, return_matching_indices=return_building_blocks or aggfunc, join_algorithm=join_algorithm, @@ -605,14 +641,14 @@ def _conditional_join_compute( df=df, right=right, conditions=conditions, - keep=keep, + keep=match_keep, ) else: indices = _get_indices_single_join._single_join( df=df, right=right, condition=conditions[0], - keep=keep, + keep=match_keep, return_matching_indices=return_building_blocks or aggfunc, ) if aggfunc and reverse: @@ -968,6 +1004,13 @@ def _create_multiindex_column(df: pd.DataFrame, right: pd.DataFrame) -> tuple: return df, right +def _get_unmatched_indices(matched_indices: np.ndarray, size: int) -> np.ndarray: + """Return row positions that are absent from the matched positions.""" + unmatched = np.ones(size, dtype=bool) + unmatched[matched_indices] = False + return unmatched.nonzero()[0] + + def _create_frame( df: pd.DataFrame, right: pd.DataFrame, @@ -982,6 +1025,8 @@ def _create_frame( """ Create final dataframe """ + df_length = len(df) + right_length = len(right) # TODO: deprecate df_columns and right_columns # user can handle column renaming before the join if (df_columns is None) and (right_columns is None): @@ -1092,9 +1137,7 @@ def _inner( include_join_positions=include_join_positions, ) if how == "left": - indexer = pd.unique(left_index) - indexer = pd.Index(indexer).get_indexer(range(len(df))) - indexer = (indexer < 0).nonzero()[0] + indexer = _get_unmatched_indices(left_index, len(df)) length = indexer.size if not length: return _inner( @@ -1138,9 +1181,7 @@ def _inner( return pd.DataFrame(dictionary, copy=False) if how == "right": - indexer = pd.unique(right_index) - indexer = pd.Index(indexer).get_indexer(range(len(right))) - indexer = (indexer < 0).nonzero()[0] + indexer = _get_unmatched_indices(right_index, len(right)) length = indexer.size if not length: return _inner( @@ -1182,13 +1223,43 @@ def _inner( value = concat_compat([arr1, arr2]) dictionary[name] = value return pd.DataFrame(dictionary, copy=False) + if how in {"left_anti", "right_anti"}: + if how == "left_anti": + indexer = _get_unmatched_indices(left_index, df_length) + else: + indexer = _get_unmatched_indices(right_index, right_length) + length = indexer.size + dictionary = {} + for key, value in df.items(): + array = value._values + if how == "left_anti": + value = array[indexer] + else: + value = construct_1d_array_from_inferred_fill_value( + value=array[:1], length=length + ) + dictionary[key] = value + for key, value in right.items(): + array = value._values + if how == "right_anti": + value = array[indexer] + else: + value = construct_1d_array_from_inferred_fill_value( + value=array[:1], length=length + ) + dictionary[key] = value + if indicator: + name, arr = _add_indicator( + indicator=indicator, + how="left" if how == "left_anti" else "right", + column_length=length, + columns=df.columns.union(right.columns), + ) + dictionary[name] = arr + return pd.DataFrame(dictionary, copy=False) # how == 'outer' - left_indexer = pd.unique(left_index) - left_indexer = pd.Index(left_indexer).get_indexer(range(len(df))) - left_indexer = (left_indexer < 0).nonzero()[0] - right_indexer = pd.unique(right_index) - right_indexer = pd.Index(right_indexer).get_indexer(range(len(right))) - right_indexer = (right_indexer < 0).nonzero()[0] + left_indexer = _get_unmatched_indices(left_index, len(df)) + right_indexer = _get_unmatched_indices(right_index, len(right)) df_nulls_length = left_indexer.size right_nulls_length = right_indexer.size diff --git a/pixi.lock b/pixi.lock index 8b8f1a44c..9c2eca750 100644 --- a/pixi.lock +++ b/pixi.lock @@ -22775,8 +22775,8 @@ packages: timestamp: 1774796815820 - pypi: ./ name: pyjanitor - version: 0.32.23 - sha256: 2c551db6e9b84114e74ba359a19fafa89130d1d5ce2b322f906d8b934e72926a + version: 0.32.24 + sha256: 048f055a9e41a71f61809b58114fe6dd79913a4232e76f8a5cf2e4b661ff05c5 requires_dist: - pandas>=3.0.0 - natsort>=8.4.0,<9 diff --git a/tests/functions/test_conditional_join.py b/tests/functions/test_conditional_join.py index 36f777997..03086f91e 100644 --- a/tests/functions/test_conditional_join.py +++ b/tests/functions/test_conditional_join.py @@ -267,6 +267,124 @@ def test_check_how_value(dummy, series): dummy.conditional_join(series, ("id", "B", "<"), how="INNER") +def test_left_anti_range_join(): + """Return left rows that do not match all range conditions.""" + events = pd.DataFrame({"time": [1, 5, 10]}) + windows = pd.DataFrame({"start": [0, 6], "end": [2, 8]}) + + actual = events.conditional_join( + windows, + ("time", "start", ">="), + ("time", "end", "<="), + how="left_anti", + indicator=True, + ) + + expected = pd.DataFrame( + { + "time": [5, 10], + "start": [np.nan, np.nan], + "end": [np.nan, np.nan], + "_merge": pd.Categorical( + ["left_only", "left_only"], + categories=["left_only", "right_only", "both"], + ), + } + ) + assert_frame_equal(expected, actual) + + +def test_right_anti_range_join(): + """Return right rows that do not match all range conditions.""" + events = pd.DataFrame({"time": [1, 5, 10]}) + windows = pd.DataFrame({"start": [0, 6], "end": [2, 8]}) + + actual = events.conditional_join( + windows, + ("time", "start", ">="), + ("time", "end", "<="), + how="right_anti", + indicator="source", + ) + + expected = pd.DataFrame( + { + "time": [np.nan], + "start": [6], + "end": [8], + "source": pd.Categorical( + ["right_only"], categories=["left_only", "right_only", "both"] + ), + } + ) + assert_frame_equal(expected, actual) + + +@pytest.mark.parametrize("how", ["left_anti", "right_anti"]) +def test_anti_join_all_rows_match(how): + """Return an empty frame with stable columns when every row matches.""" + left = pd.DataFrame({"left": [1, 2]}) + right = pd.DataFrame({"right": [2]}) + + actual = left.conditional_join(right, ("left", "right", "<="), how=how) + + assert actual.empty + assert actual.columns.tolist() == ["left", "right"] + + +def test_left_anti_equi_and_non_equi_join(): + """Support anti joins with combined equi and non-equi conditions.""" + left = pd.DataFrame({"group": [1, 1, 2], "value": [1, 5, 3]}) + right = pd.DataFrame({"group": [1, 2], "limit": [2, 4]}) + + actual = left.conditional_join( + right, + ("group", "group", "=="), + ("value", "limit", "<"), + how="left_anti", + df_columns=["group", "value"], + right_columns=None, + ) + + expected = left.iloc[[1]].reset_index(drop=True) + assert_frame_equal(expected, actual) + + +@pytest.mark.parametrize("keep", ["first", "last"]) +def test_right_anti_ignores_keep(keep): + """Consider every matching right row when filtering a right anti join.""" + left = pd.DataFrame({"left": [0]}) + right = pd.DataFrame({"right": [1, 2]}) + + actual = left.conditional_join( + right, ("left", "right", "<"), how="right_anti", keep=keep + ) + + assert actual.empty + + +@pytest.mark.parametrize( + ("how", "df_columns", "right_columns", "match"), + [ + ("left_anti", None, slice(None), "df_columns cannot be None"), + ("right_anti", slice(None), None, "right_columns cannot be None"), + ], +) +def test_anti_join_preserved_columns_required(how, df_columns, right_columns, match): + """Require output columns from the side preserved by an anti join.""" + left = pd.DataFrame({"left": [1]}) + right = pd.DataFrame({"right": [0]}) + + with pytest.raises(ValueError, match=match): + left.conditional_join( + right, + ("left", "right", "<"), + how=how, + df_columns=df_columns, + right_columns=right_columns, + ) + + def test_check_use_numba_type(dummy, series): """ Raise TypeError if `use_numba` is not a boolean. From 0af0ef641e65c0c4e7245075d6be4e96f3936984 Mon Sep 17 00:00:00 2001 From: samukweku Date: Wed, 19 Aug 2026 17:31:38 +1000 Subject: [PATCH 2/4] [DOC] correct anti join version annotation --- janitor/functions/conditional_join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/janitor/functions/conditional_join.py b/janitor/functions/conditional_join.py index 9c3249583..f324fc706 100644 --- a/janitor/functions/conditional_join.py +++ b/janitor/functions/conditional_join.py @@ -284,7 +284,7 @@ def conditional_join( - 0.32.10 - Added `include_join_positions` parameter. - Added `join_algorithm` parameter. - - 0.33.0 + - 0.32.25 - Added `left_anti` and `right_anti` joins. Args: From a595c9f90d6f959c285b95ec85283bf7a9a45062 Mon Sep 17 00:00:00 2001 From: samukweku Date: Wed, 19 Aug 2026 17:33:37 +1000 Subject: [PATCH 3/4] [DOC] document API version annotations --- AGENTS.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 6c1a60f3b..9f9f30c82 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -567,6 +567,19 @@ mental model in the code documentation as well as technical API details. nearby documentation. Do not add redundant ELI5 comments to straightforward helpers. +### [2026-08-19] Derive API Documentation Versions from the Release Sequence + +**Context**: Adding a `Version Changed` entry for a feature while other +docstrings referenced a future minor release for deprecation. +**Learning**: An unrelated deprecation target is not evidence for the version +that will introduce a new feature. The package version and active release +sequence determine the annotation. +**Recommendation**: Before adding `Version Added`, `Version Changed`, or +deprecation metadata, check the current version in `pyproject.toml` and recent +release commits. Use the next patch version for ordinary unreleased changes +unless a maintainer or release plan specifies a minor or major release. If the +release target remains ambiguous, ask rather than copying another annotation. + --- ## Version History From 1416b3ea56646eff1cc0c9d64f967f045e4ba059 Mon Sep 17 00:00:00 2001 From: samukweku Date: Wed, 19 Aug 2026 23:15:55 +1000 Subject: [PATCH 4/4] docs: add changelog issue and PR attribution --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3eb7b1c2..a841bef7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - [ENH] Replace `.apply()` with `.map()` in `find_replace` for ~6x performance improvement. - Issue #1422 @Anupam2400 - [ENH] Add `drop_first` parameter to `expand_column`. - Issue #368 @manav252 - [ENH] Add `left_anti` and `right_anti` joins to `conditional_join`. - - Issue #1627 @samukweku + Issue #1627, PR #1633 @samukweku - [ENH] Add `include_join_positions` parameter to `conditional_join`; added limited support for join aggregations via the `join_agg` function. - Issue #1497 @samukweku - [ENH] Added `rle_id` function for run-length encoding IDs - Issue #1435 @emmanuel-ferdman - [ENH] Add `scale_mad` for robust median/MAD scaling. - PR #1530 @ShachiMistry