From 52064d3dc015834454c417c1bc4e76ed48cc8824 Mon Sep 17 00:00:00 2001 From: stephenholleran Date: Thu, 2 Apr 2026 17:27:24 +0100 Subject: [PATCH 1/5] prep for next release --- CHANGELOG.md | 13 +++++++++++++ brightwind/__init__.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4882b3c..2b1019fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,19 @@ Given a version number MAJOR.MINOR.PATCH, increment the: Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format. +--- +## [2.X.X] +XX-Xxx-2026 + +### New Features and Enhancements +1. + +### Bug Fixes +1. + +### Deprecated +1. + --- ## [2.6.0] 02-Apr-2026 diff --git a/brightwind/__init__.py b/brightwind/__init__.py index bfd7247b..7c43d7f9 100644 --- a/brightwind/__init__.py +++ b/brightwind/__init__.py @@ -34,4 +34,4 @@ __all__ = ['analyse', 'transform', 'export', 'load', 'demo_datasets'] -__version__ = '2.6.0' +__version__ = '2.7.0_dev' From 1942bf3982abd84301e158247cdec9d130e3c3aa Mon Sep 17 00:00:00 2001 From: dancasey-ie Date: Wed, 13 May 2026 17:01:14 +0100 Subject: [PATCH 2/5] Iss601 read brighthub timeseries as parquet (#608) * iss #601 add parquet support to LoadBrightHub.get_data Adds a file_extension argument ('.csv' or '.parquet') to LoadBrightHub.get_data so BrightHub timeseries can be fetched as parquet for faster downloads and preserved dtypes. The current '.csv' default is retained for backwards compatibility but now emits a DeprecationWarning; '.parquet' is expected to become the default in the next major release. Parquet reads use pd.read_parquet with engine='auto'; pyarrow and fastparquet are exposed as optional extras in setup.py. Co-Authored-By: Claude Opus 4.7 (1M context) * Update CHANGELOG.md with version number --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: stephenholleran --- CHANGELOG.md | 8 ++-- brightwind/load/load.py | 62 +++++++++++++++++++++++---- setup.py | 6 +++ tests/test_load.py | 93 +++++++++++++++++++++++++++++++++++++++-- 4 files changed, 154 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b1019fc..0562c5c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,17 +10,17 @@ Given a version number MAJOR.MINOR.PATCH, increment the: Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format. --- -## [2.X.X] -XX-Xxx-2026 +## [2.7.0] +XX-May-2026 ### New Features and Enhancements -1. +1. Added support for reading Parquet files from BrightHub in `LoadBrightHub.get_data()` via a new `file_extension` argument (one of `'.csv'` or `'.parquet'`). Default remains `'.csv'` for backwards compatibility; Parquet is expected to become the default in the next major release. Reading Parquet requires a parquet engine — install with `pip install brightwind[parquet]` (pyarrow) or `pip install brightwind[parquet-fastparquet]` (fastparquet). ([#601](https://github.com/brightwind-dev/brightwind/issues/601)) ### Bug Fixes 1. ### Deprecated -1. +1. The implicit default of `file_extension='.csv'` in `LoadBrightHub.get_data()` is deprecated and will change to `'.parquet'` in the next major release. Calls that do not pass `file_extension` explicitly will emit a `DeprecationWarning`. Pass `file_extension='.csv'` to keep the current behaviour, or `file_extension='.parquet'` to opt in early. ([#601](https://github.com/brightwind-dev/brightwind/issues/601)) --- ## [2.6.0] diff --git a/brightwind/load/load.py b/brightwind/load/load.py index bcf5a8e6..965b1586 100644 --- a/brightwind/load/load.py +++ b/brightwind/load/load.py @@ -9,7 +9,7 @@ import os import shutil import json -from io import StringIO +from io import StringIO, BytesIO import warnings from dateutil.parser import parse from brightwind.analyse import plot as bw_plt @@ -31,6 +31,8 @@ 'apply_cleaning_rules', 'apply_cleaning_windographer'] +_FILE_EXTENSION_NOT_SET = object() + OPERATOR_DICT = { 1: operator.lt, 2: operator.le, @@ -1491,19 +1493,19 @@ def __date_to_datetime_str(date_str): return date_str @staticmethod - def __get_timeseries_data(measurement_station_uuid, date_from=None, date_to=None): + def __get_timeseries_data(measurement_station_uuid, date_from=None, date_to=None, file_extension='.csv'): """ Sub function to return the Brighthub GET timeseries-data API response. """ date_from = LoadBrightHub.__date_to_datetime_str(date_from) date_to = LoadBrightHub.__date_to_datetime_str(date_to) - + return LoadBrightHub._brighthub_request( url_end=f"/measurement-locations/{measurement_station_uuid}/timeseries-data", - params={"date_from": date_from, "date_to": date_to}) + params={"date_from": date_from, "date_to": date_to, "file_extension": file_extension}) @staticmethod - def get_data(measurement_station_uuid, date_from=None, date_to=None): + def get_data(measurement_station_uuid, date_from=None, date_to=None, file_extension=_FILE_EXTENSION_NOT_SET): """ Get the timeseries data from BrightHub for a particular measurement station. @@ -1516,6 +1518,17 @@ def get_data(measurement_station_uuid, date_from=None, date_to=None): :type date_from: str :param date_to: Optional filter to retrieve data up to this date. :type date_to: str + :param file_extension: File format to request from BrightHub. One of '.csv' (current default) or + '.parquet'. Reading '.parquet' requires either ``pyarrow`` or ``fastparquet`` + to be installed (``pip install brightwind[parquet]`` for ``pyarrow``, or + ``pip install brightwind[parquet-fastparquet]`` for ``fastparquet``). + + .. deprecated:: + The default of '.csv' is retained for backwards compatibility and will + change to '.parquet' in the next major release of brightwind. Pass + ``file_extension`` explicitly to silence the DeprecationWarning and lock + in the desired format. + :type file_extension: str :return: The timeseries data. :rtype: pd.DataFrame @@ -1544,8 +1557,31 @@ def get_data(measurement_station_uuid, date_from=None, date_to=None): data = bw.LoadBrightHub.get_data(measurement_station_uuid='9344e576-6d5a-45f0-9750-2a7528ebfa14', date_to='2016-07-01') + To get data as a parquet file (faster downloads and preserves dtypes; requires a parquet engine to be + installed) + :: + data = bw.LoadBrightHub.get_data(measurement_station_uuid='9344e576-6d5a-45f0-9750-2a7528ebfa14', + file_extension='.parquet') + """ - response = LoadBrightHub.__get_timeseries_data(measurement_station_uuid, date_from, date_to) + if file_extension is _FILE_EXTENSION_NOT_SET: + warnings.warn( + "The default file_extension for LoadBrightHub.get_data will change from '.csv' to '.parquet' in the " + "next major release of brightwind. Pass file_extension='.csv' explicitly to keep the current " + "behaviour, or file_extension='.parquet' to opt in early. Reading parquet requires a parquet engine: " + "`pip install brightwind[parquet]` (pyarrow) or `pip install brightwind[parquet-fastparquet]` " + "(fastparquet).", + DeprecationWarning, + stacklevel=2, + ) + file_extension = '.csv' + + if file_extension not in ('.csv', '.parquet'): + raise ValueError( + f"Unsupported file_extension '{file_extension}'. Expected '.csv' or '.parquet'." + ) + + response = LoadBrightHub.__get_timeseries_data(measurement_station_uuid, date_from, date_to, file_extension) response_json = response.json() # Handle 503 Service Unavailable error response using Retry-After header. @@ -1556,7 +1592,9 @@ def get_data(measurement_station_uuid, date_from=None, date_to=None): retry_after = int(response.headers.get('Retry-After', 60)) # default retry after default 60 seconds for _ in range(4): # attempt 4 more times time.sleep(retry_after) # wait before retrying - response = LoadBrightHub.__get_timeseries_data(measurement_station_uuid, date_from, date_to) + response = LoadBrightHub.__get_timeseries_data( + measurement_station_uuid, date_from, date_to, file_extension + ) response_json = response.json() if response.status_code == 503 and assembly_in_progress_err_msg in response_json.get('details'): retry_after *= 2 # double the retry time for each attempt @@ -1581,6 +1619,16 @@ def get_data(measurement_station_uuid, date_from=None, date_to=None): # Handle all request-related errors raise RuntimeError(f"An error occurred while fetching the data: {e}") + if file_extension == '.parquet': + try: + return pd.read_parquet(BytesIO(timeseries_response.content), engine='auto') + except ImportError as e: + raise ImportError( + "Reading parquet requires a parquet engine. Install one with " + "`pip install brightwind[parquet]` (pyarrow) or " + "`pip install brightwind[parquet-fastparquet]` (fastparquet)." + ) from e + df = pd.read_csv(StringIO(timeseries_response.text)) df['Timestamp'] = pd.to_datetime(df['Timestamp']) # this throws error if return doesn't have 'Timestamp' return df.set_index('Timestamp') diff --git a/setup.py b/setup.py index 9b17df79..5f814370 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,12 @@ def get_version(rel_path): 'jsonschema>=4.17.3', 'jinja2>= 3.0.0' ], + extras_require={ + # Optional parquet engines for LoadBrightHub.get_data(file_extension='.parquet'). + # pandas picks the installed engine at read time (prefers pyarrow, falls back to fastparquet). + 'parquet': ['pyarrow>=5.0.0'], + 'parquet-fastparquet': ['fastparquet>=0.8.0'], + }, classifiers=[ "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", diff --git a/tests/test_load.py b/tests/test_load.py index 7332d240..eebf0eaf 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -1,10 +1,12 @@ import pytest import brightwind as bw import os +import warnings import pandas as pd import numpy as np import json -from unittest.mock import patch +from io import BytesIO +from unittest.mock import patch, MagicMock DEMO_DATA_FOLDER = os.path.join(os.path.dirname(__file__), '../brightwind/demo_datasets') @@ -210,10 +212,17 @@ def test_load_brighthub(): ) == test_period_demo_data # To get data for a specific time period for a specific measurement station - data_columns = bw.LoadBrightHub.get_data(measurement_station_uuid=measurement_station_uuid, date_from='2016-12-01', - date_to='2017-01-01').columns + data_csv = bw.LoadBrightHub.get_data(measurement_station_uuid=measurement_station_uuid, date_from='2016-12-01', + date_to='2017-01-01', file_extension='.csv') for col in ['Spd80mN', 'Spd80mS', 'Dir78mS']: - assert col in data_columns + assert col in data_csv.columns + + # Same window as parquet — confirms the API accepts file_extension='.parquet' and returns equivalent data. + data_parquet = bw.LoadBrightHub.get_data(measurement_station_uuid=measurement_station_uuid, date_from='2016-12-01', + date_to='2017-01-01', file_extension='.parquet') + assert list(data_parquet.columns) == list(data_csv.columns) + assert len(data_parquet) == len(data_csv) + assert data_parquet.index.name == 'Timestamp' # To get cleaning log cleaning_log_df = bw.LoadBrightHub.get_cleaning_log(measurement_station_uuid=measurement_station_uuid) @@ -222,3 +231,79 @@ def test_load_brighthub(): # To get cleaning rules cleaning_rules_json = bw.LoadBrightHub.get_cleaning_rules(measurement_station_uuid=measurement_station_uuid) assert cleaning_rules_json[0]['measurement_location_uuid'] == measurement_station_uuid + + +def test_load_brighthub_get_data_default_emits_deprecation_warning(): + api_response = MagicMock(status_code=200) + api_response.json.return_value = {'url': 'https://example.com/presigned'} + csv_response = MagicMock(text='Timestamp,Spd80mN\n2016-06-01 00:00:00,7.1\n') + + with patch('brightwind.load.load.LoadBrightHub._brighthub_request', return_value=api_response) as mock_request, \ + patch('brightwind.load.load.requests.get', return_value=csv_response): + with pytest.warns(DeprecationWarning, match="file_extension"): + df = bw.LoadBrightHub.get_data(measurement_station_uuid='uuid-1') + + assert mock_request.call_args.kwargs['params']['file_extension'] == '.csv' + assert df.index.name == 'Timestamp' + + +def test_load_brighthub_get_data_csv_explicit_no_warning(): + api_response = MagicMock(status_code=200) + api_response.json.return_value = {'url': 'https://example.com/presigned'} + csv_response = MagicMock(text='Timestamp,Spd80mN\n2016-06-01 00:00:00,7.1\n') + + with patch('brightwind.load.load.LoadBrightHub._brighthub_request', return_value=api_response) as mock_request, \ + patch('brightwind.load.load.requests.get', return_value=csv_response): + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + df = bw.LoadBrightHub.get_data(measurement_station_uuid='uuid-1', file_extension='.csv') + + assert mock_request.call_args.kwargs['params']['file_extension'] == '.csv' + assert isinstance(df.index, pd.DatetimeIndex) + + +def test_load_brighthub_get_data_parquet(): + pytest.importorskip('pyarrow', reason="parquet engine required to round-trip parquet bytes") + + index = pd.to_datetime(['2016-06-01 00:00:00', '2016-06-01 00:10:00']) + index.name = 'Timestamp' + sample = pd.DataFrame({'Spd80mN': [7.1, 7.3]}, index=index) + buffer = BytesIO() + sample.to_parquet(buffer, compression='snappy') + + api_response = MagicMock(status_code=200) + api_response.json.return_value = {'url': 'https://example.com/presigned'} + parquet_response = MagicMock(content=buffer.getvalue()) + + with patch('brightwind.load.load.LoadBrightHub._brighthub_request', return_value=api_response) as mock_request, \ + patch('brightwind.load.load.requests.get', return_value=parquet_response): + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + df = bw.LoadBrightHub.get_data(measurement_station_uuid='uuid-1', file_extension='.parquet') + + assert mock_request.call_args.kwargs['params']['file_extension'] == '.parquet' + assert df.index.name == 'Timestamp' + assert isinstance(df.index, pd.DatetimeIndex) + assert np.allclose(df['Spd80mN'].values, sample['Spd80mN'].values) + + +def test_load_brighthub_get_data_invalid_extension(): + with patch('brightwind.load.load.LoadBrightHub._brighthub_request') as mock_request, \ + patch('brightwind.load.load.requests.get') as mock_get: + with pytest.raises(ValueError, match="Unsupported file_extension"): + bw.LoadBrightHub.get_data(measurement_station_uuid='uuid-1', file_extension='.xlsx') + + mock_request.assert_not_called() + mock_get.assert_not_called() + + +def test_load_brighthub_get_data_parquet_missing_engine_raises_importerror(): + api_response = MagicMock(status_code=200) + api_response.json.return_value = {'url': 'https://example.com/presigned'} + parquet_response = MagicMock(content=b'irrelevant') + + with patch('brightwind.load.load.LoadBrightHub._brighthub_request', return_value=api_response), \ + patch('brightwind.load.load.requests.get', return_value=parquet_response), \ + patch('brightwind.load.load.pd.read_parquet', side_effect=ImportError("no engine")): + with pytest.raises(ImportError, match=r"brightwind\[parquet\].*brightwind\[parquet-fastparquet\]"): + bw.LoadBrightHub.get_data(measurement_station_uuid='uuid-1', file_extension='.parquet') From f9a432d444d937fc61ea9ab689399f6dadee815b Mon Sep 17 00:00:00 2001 From: dancasey-ie Date: Thu, 14 May 2026 11:31:49 +0100 Subject: [PATCH 3/5] Iss602 update contributing docs for test envs (#603) * iss #602 align README and contributing.md with the docs site Trims duplicated content (Features, Benefits, Test datasets) from README.md now that the same material lives on the docs site, and restructures contributing.md as a contributor-focused guide. Adds venv/conda install options to both, a Jupyter quick-start example to README.md, and dev-environment + test-running instructions to contributing.md. Co-Authored-By: Claude Opus 4.7 (1M context) * Include potentional venv folders * iss #602 reframe install options, add Why open-source? section, gitignore venvs Reframes the README install options so venv is presented as the quick try-it-out path and conda as the option for Anaconda/Jupyter users, drops the "recommended for new Python users" label, and adds uv as a third option. Adds a short "Why open-source?" section covering transparency, auditability and the shared-toolkit goal. Adds .venv/, venv/ and env/ to .gitignore so in-repo virtual environments are not committed. Co-Authored-By: Claude Opus 4.7 (1M context) * reintroduce datset table * iss #602 skip BrightHub integration test when credentials are missing Adds a pytest.mark.skipif to test_load_brighthub so the live BrightHub API test is skipped (with a clear reason) when BRIGHTHUB_CLIENT_ID or BRIGHTHUB_CLIENT_SECRET are not set in the environment, instead of failing on auth. The mocked unit tests added earlier still run unconditionally. Also flags the fork-based contribution workflow in contributing.md and documents how to set the BrightHub credentials on Windows and macOS/Linux for contributors who want to run the integration test locally. Co-Authored-By: Claude Opus 4.7 (1M context) * iss #602 add changelog entries for docs rewrite and BrightHub test skip Co-Authored-By: Claude Opus 4.7 (1M context) * Update README.md * iss #602 also accept BRIGHTHUB_EMAIL/PASSWORD for BrightHub test skip Co-Authored-By: Claude Opus 4.7 (1M context) * iss #602 document optional parquet extras and accept either parquet engine in test Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: stephenholleran --- .gitignore | 7 +- CHANGELOG.md | 2 + README.md | 147 +++++++++++++++++++------------ contributing.md | 215 ++++++++++++++++++++++++++++++++++++++------- tests/test_load.py | 13 ++- 5 files changed, 295 insertions(+), 89 deletions(-) diff --git a/.gitignore b/.gitignore index b6b1f093..229a4e9f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,9 @@ log/ tests/temp/* temp/* brightwind.egg-info/ -**/.ipynb_checkpoints \ No newline at end of file +**/.ipynb_checkpoints + +### Virtual Environments ### +.venv/ +venv/ +env/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 0562c5c7..6245a5be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ Additional labels for pre-release and build metadata are available as extensions XX-May-2026 ### New Features and Enhancements +1. Rewrote `README.md` and `contributing.md` to align with the [docs site](https://brightwind-dev.github.io/brightwind-docs/), with clearer install options (venv / conda), a quick-start example, and an expanded contributing guide covering the fork workflow, editable dev install and running the test suite. ([#602](https://github.com/brightwind-dev/brightwind/issues/602)) +2. `test_load_brighthub` is now skipped automatically when no BrightHub credentials are available — either `BRIGHTHUB_CLIENT_ID` and `BRIGHTHUB_CLIENT_SECRET`, or the deprecated `BRIGHTHUB_EMAIL` and `BRIGHTHUB_PASSWORD` — so the test suite runs cleanly without BrightHub credentials. ([#602](https://github.com/brightwind-dev/brightwind/issues/602)) 1. Added support for reading Parquet files from BrightHub in `LoadBrightHub.get_data()` via a new `file_extension` argument (one of `'.csv'` or `'.parquet'`). Default remains `'.csv'` for backwards compatibility; Parquet is expected to become the default in the next major release. Reading Parquet requires a parquet engine — install with `pip install brightwind[parquet]` (pyarrow) or `pip install brightwind[parquet-fastparquet]` (fastparquet). ([#601](https://github.com/brightwind-dev/brightwind/issues/601)) ### Bug Fixes diff --git a/README.md b/README.md index cad766c3..24f68f57 100644 --- a/README.md +++ b/README.md @@ -13,106 +13,139 @@
-Brightwind is a Python library specifically built for wind analysis. It can load in wind speed, wind direction and -other metrological timeseries data. There are various plots you can use to understand this data and to find any -potential issues. You can perform many common functions to the data such as shear and long-term adjustments. The -resulting adjusted data is then outputted as a frequency distribution tab file which can be used in wind analysis -software such as WAsP. +Brightwind is an open-source Python library for wind (and solar) resource analysis. It loads meteorological +timeseries data, runs common analyses — shear, long-term adjustments, correlations, distributions — and exports +results to formats used by wind analysis software such as WAsP. -This library can also be used for solar resource analysis. +📚 **Full documentation, tutorials and API reference:** https://brightwind-dev.github.io/brightwind-docs/
--- ### Installation -You can use pip from the command line to install the library. +Install brightwind into its own environment to avoid dependency clashes. Pick whichever option suits you. +#### Option 1 — venv (quick way to try brightwind out) + +Check Python is installed (3.9+ recommended): + +```bash +python --version ``` -C:\Users\Stephen> pip install brightwind -``` -It is advisable to use a separate environment to avoid any dependency clashes with other libraries such as Pandas, Numpy -or Matplotlib you may already have installed. -
+If not, install it from [python.org/downloads](https://www.python.org/downloads/) — on Windows, tick **"Add +Python to PATH"** in the installer. -For those that do not have Python installed and are just getting started, we recommend installing Anaconda. Anaconda is -a Python distribution for scientific computing and so provides everything you need, Python, pip and Jupyter Notebook -along with libraries such as Pandas, Numpy and Matplotlib. Datacamp provide a good tutorial for [installing -Anaconda on Windows](https://www.datacamp.com/tutorial/installing-anaconda-windows) to get started. +Then create an environment and install brightwind: -Once Anaconda is installed, you can use the **Anaconda Prompt** to run the above command line `pip install brightwind`. -Or first use **Anaconda Navigator** to create an environment. +```bash +python -m venv brightwind_env +# Windows +brightwind_env\Scripts\activate +# macOS / Linux +source brightwind_env/bin/activate ---- -### Documentation +pip install brightwind +``` -Documentation on how to get setup and use the library can be found at https://brightwind-dev.github.io/brightwind-docs/ +#### Option 2 — conda -
+Common if you already use Anaconda or work primarily in Jupyter. +[Anaconda](https://www.datacamp.com/tutorial/installing-anaconda-windows) bundles Python, pip and Jupyter in one +installer. From the **Anaconda Prompt**: -Example usage of the brightwind library is shown below using Jupyter Notebook. Jupyter Notebook is a powerful way to -immediately see the results of code you have written. -
+```bash +conda create --name brightwind_env python=3.11 +conda activate brightwind_env +pip install brightwind +``` -

+A step-by-step Windows install walkthrough is also available in the +[tutorials](https://brightwind-dev.github.io/brightwind-docs/). -![demo_image_1](read_me_1.png) -![demo_image_2](read_me_2.png) -

+**Optional — parquet support** +Reading `.parquet` files from BrightHub needs a parquet engine. Install the default (pyarrow): +```bash +pip install brightwind[parquet] +``` +Or use fastparquet instead: `pip install brightwind[parquet-fastparquet]`.
-##### Features -The library provides wind analysts with easy to use tools for working with -meteorological data. It supports loading of meteorological data, averaging, -filtering, plotting, correlations, shear analysis, long term adjustments, etc. -The library can then export a resulting long term adjusted tab file to be used in -other wind analysis software. +--- +### Quick start + +Most analysts use brightwind from a Jupyter Notebook: + +```bash +pip install jupyter +jupyter notebook +``` + +```python +import brightwind as bw + +data = bw.load_csv(bw.demo_datasets.demo_data) +bw.basic_stats(data) +``` + +For full examples — loading, plotting, shear, correlations, exporting — see the +[tutorials and API reference](https://brightwind-dev.github.io/brightwind-docs/).
-##### Benefits -The key benefits to an open-source library is that it provides complete transparency -and traceability. Anyone in the industry can review any part of the code and suggest changes, -thus creating a standardised, validated toolkit for the industry. +

-By default, during an assessment every manipulation or adjustment made to the wind data is -contained in a single file. This can easily be reviewed and checked by internal reviewers or, -as the underlying code is open-sourced, there is no reason why this file cannot be sent to -3rd parties for review thus increasing the effectiveness of a banks due diligence. +![demo_image_1](read_me_1.png) +![demo_image_2](read_me_2.png) +


-##### License -The library is licensed under the MIT license. +--- +### Why open-source? + +The brightwind library is open-source, making every step of an assessment transparent, auditable and reproducible. The full record of +adjustments to a dataset lives in a single file that internal reviewers, third parties and banks can inspect +directly — sharpening due diligence and removing the "black box" problem of proprietary tools. + +The intent is a shared, validated toolkit that the wind and solar industry builds on together, rather than each +consultancy reinventing the same calculations behind closed doors.
--- ### Test datasets -A test dataset is included in this repository and is used to demonstrate function and test functions in the code. -Other files and datasets are also included to complement this demo dataset. These are outlined below: -
+Demo datasets are bundled with the library to demonstrate functions and exercise the test suite: -| Dataset | Source | Notes | -|:--------------------- |:-------------|:-----| -| demo_data.csv | BrightWind | A modified 2 year met mast dataset in csv and Campbell Scientific format. | -| MERRA-2_XX_2000-01-01_2017-06-30.csv | NASA [GES DISC](https://disc.gsfc.nasa.gov/) | 4 x MERRA-2 18-yr datasets to complement the demo data for long term analyses. | -| demo_cleaning_file.csv | BrightWind | A file containing information on what periods to clean out from the demo data. | -| windographer_flagging_log.txt | BrightWind | The same cleaning info as found in 'demo_cleaning_file.csv' formatted as a Windographer flagging file. | -| demo_data_iea43_wra_data_model.json | BrightWind | A JSON file formatted according to the IEA Wind Task 43 [WRA Data Model](https://github.com/IEA-Task-43/digital_wra_data_standard) standard which describes the mast configuration for the demo data. | +| Dataset | Source | Notes | +|:--- |:--- |:--- | +| `demo_data.csv` | BrightWind | A modified 2-year met mast dataset in CSV and Campbell Scientific format. | +| `MERRA-2_XX_2000-01-01_2017-06-30.csv` | NASA [GES DISC](https://disc.gsfc.nasa.gov/) | 4 × MERRA-2 18-year datasets to complement the demo data for long-term analyses. | +| `demo_cleaning_file.csv` | BrightWind | Periods to clean out from the demo data. | +| `windographer_flagging_log.txt` | BrightWind | Same cleaning info as `demo_cleaning_file.csv` formatted as a Windographer flagging file. | +| `demo_data_iea43_wra_data_model.json` | BrightWind | A JSON file formatted to the IEA Wind Task 43 [WRA Data Model](https://github.com/IEA-Task-43/digital_wra_data_standard) standard, describing the mast configuration for the demo data. |
--- ### Contributing -If you wish to be involved or find out more please contact stephen@brightwindanalysis.com. -More information can be found in the [contributing.md](https://github.com/brightwind-dev/brightwind/blob/master/contributing.md) section of the website. +Brightwind welcomes contributions from across the wind and solar industry — analysts, engineers, researchers and +developers. + +- **Issues, bugs and feature requests:** [GitHub issue tracker](https://github.com/brightwind-dev/brightwind/issues) +- **Code contributions and development setup:** see [contributing.md](contributing.md) +- **General enquiries:** stephen@brightwindanalysis.com
+ +--- +### License + +MIT — see [LICENSE.txt](LICENSE.txt). diff --git a/contributing.md b/contributing.md index 7841e819..2157a8a9 100644 --- a/contributing.md +++ b/contributing.md @@ -1,48 +1,203 @@ -Contributing -============ +# Contributing -## Issue Tracking +Brightwind is an **open-source library** and contributions from the wind and solar industry are warmly welcomed — +analysts, engineers, researchers and developers. Bug fixes, new features, docs improvements and feedback all help +build a standardised, validated toolkit for the industry. -New feature requests, changes, enhancements, non-methodology features, and bug reports can be filed as new issues in the -[GitHub.com issue tracker](https://github.com/brightwind-dev/brightwind/issues) at any time. Please be sure to fully describe the -issue. +For library usage and tutorials, see the [documentation site](https://brightwind-dev.github.io/brightwind-docs/). +This guide covers contributing to the **codebase itself**. +--- -## Repository +## Reporting issues -The brightwind repository is hosted on GitHub, and located here: https://github.com/brightwind-dev/brightwind +Search the [issue tracker](https://github.com/brightwind-dev/brightwind/issues) before opening a new issue. When +reporting a bug or requesting a feature, please include: -This repository is organized using a modified git-flow system. Branches are organized as follows: +- A clear, descriptive title. +- What you expected to happen, and what actually happened. +- A minimal code example that reproduces the problem. +- Your brightwind version, Python version and operating system. -- master: Stable release version. Must have good test coverage and may not have all the newest features. -- dev: Development branch which contains the newest features. Tests must pass, but code may be unstable. -- feature/xxx: Branch from dev, should reference a GitHub issue number. +--- -To work on a feature, first create an issue exaplaining the issue or idea and clone the repository locally when you are ready to get to work. +## Development workflow -Once you have successfully cloned, create a feature branch from the dev branch. Please name your branch with the issue number you are working from e.g. `iss255_fix_bug`. Work out of this feature branch and push changes to it before submitting a pull request. +The repo follows a modified git-flow: -Commit messages should contain the issue number, preceded with a #, so these commits can automatically show up in the issue conversation e.g. `git commit -m "iss #255 bug fixed"`. +- **`master`** — stable release. +- **`dev`** — newest features; tests must pass. +- **`iss_`** — feature branches off `dev`, one per issue. -Be sure to periodically synchronize the upstream dev branch into your feature branch to avoid conflicts in the pull request. You can continue to iterate on your feature branch until you are ready to submit your pull request. +### Fork, branch and commit -When the feature branch is ready, make a pull request to the dev branch through the GitHub.com UI. +Brightwind follows the standard open-source fork-based workflow. -## Pull Request +> **Internal BrightWind contributors** with push access to `brightwind-dev/brightwind` can skip step 1 and +> clone the upstream repo directly. -Pull requests must be made for any changes to be merged into release branches. -They must include updated documentation and pass all unit tests and integration tests. -In addition, code coverage should not be negatively affected by the pull request. +1. **Fork the repo** on GitHub — click the **Fork** button (top-right of the + [brightwind repo page](https://github.com/brightwind-dev/brightwind)). -**Scope:** -Encapsulate the changes of ideally one, or potentially a couple, issues. -It is greatly preferable to submit three small pull requests than it is to submit one large pull request. -Write a complete description of these changes in the pull request body. +2. **Clone your fork** locally: + ```bash + git clone https://github.com/YOUR_USERNAME/brightwind.git + cd brightwind + ``` -**Tests:** -The contributor should write a test to cover the new changes. +3. **Add the upstream remote** so you can pull future changes from the main repo: + ```bash + git remote add upstream https://github.com/brightwind-dev/brightwind.git + ``` -**Documentation:** -Function docstrings should be fully updated to explain any behaviour changes to the user along with examples. For complicated logic some inline comments are encouraged. +4. **Sync `dev`** and create a feature branch named after the issue: + ```bash + git checkout dev + git pull upstream dev + git checkout -b iss123_short_description + ``` -**Changelog:** For pull requests that encapsulate a user-facing feature, or is significant to other contributors for some other reason, please add a line to CHANGELOG.md in the [Unreleased] section. +5. Keep changes small and focused. Verify before committing: + ```bash + git diff + git status + ``` + +6. **Commit** with a message that **starts with the issue number** (the space between `iss` and `#` links the + commit to the issue on GitHub): + ```bash + git commit -m "iss #123 short description of the change" + ``` + +7. **Push to your fork**: + ```bash + git push -u origin iss123_short_description + ``` + +### Pull requests + +Open PRs from your fork's feature branch into **`brightwind-dev/brightwind:dev`** — not `master`. + +Before raising a PR: + +- All tests pass — see [Running tests](#running-tests). +- Add a [CHANGELOG.md](CHANGELOG.md) entry under `[Unreleased]` for user-facing changes. +- Update docstrings for new or changed behaviour. +- Add a test for new functionality or bug fixes. + +In the PR, keep the scope focused (ideally one issue), reference the issue number in the title +(e.g. `[iss123] Short description`), and clearly explain what changed and why. + +--- + +## Development setup + +Install brightwind in **editable mode** in a dedicated environment. + +### Option 1 — venv + +```bash +python -m venv brightwind_dev +# Windows +brightwind_dev\Scripts\activate +# macOS / Linux +source brightwind_dev/bin/activate +``` + +### Option 2 — conda + +```bash +conda create --name brightwind_dev python=3.11 +conda activate brightwind_dev +``` + +### Editable install + +With the environment active, clone the repo and install brightwind in editable mode: + +```bash +git clone https://github.com/brightwind-dev/brightwind.git +cd brightwind +pip install -e . +``` + +The `-e` flag links the install to your local clone, so code changes are picked up without reinstalling. + +To work on or test parquet loading, include the optional engine (pyarrow by default): + +```bash +pip install -e .[parquet] +``` + +Or use fastparquet instead: `pip install -e .[parquet-fastparquet]`. + +--- + +## Running tests + +From the repo root: + +```bash +pytest tests/ +``` + +Single file or single test: + +```bash +pytest tests/test_load.py +pytest tests/test_load.py::test_load_brighthub -v +``` + +All tests must pass before raising a PR. New functionality should include tests covering normal behaviour and +edge cases. + +### Integration tests against BrightHub + +Some tests hit the live [BrightHub](https://brighthub.io) API and require API credentials. They are automatically +**skipped** if the credentials are not set, so the rest of the suite runs cleanly without them. + +Tests covering `.parquet` downloads also require the `[parquet]` extra (see [Development setup](#development-setup)) — they are skipped otherwise. + +To run the BrightHub tests, set **either** an API key pair **or** an email/password pair as environment variables +before running `pytest`. + +Using an API key (recommended — generate one from +[brighthub.io/account-settings/settings](https://brighthub.io/account-settings/settings)): + +```bash +# Windows (PowerShell, current session) +$env:BRIGHTHUB_CLIENT_ID="your-client-id" +$env:BRIGHTHUB_CLIENT_SECRET="your-client-secret" + +# macOS / Linux +export BRIGHTHUB_CLIENT_ID="your-client-id" +export BRIGHTHUB_CLIENT_SECRET="your-client-secret" +``` + +Using email and password (deprecated — will be removed in the next major release (v3.0); not available for +Microsoft SSO users): + +```bash +# Windows (PowerShell, current session) +$env:BRIGHTHUB_EMAIL="your-email" +$env:BRIGHTHUB_PASSWORD="your-password" + +# macOS / Linux +export BRIGHTHUB_EMAIL="your-email" +export BRIGHTHUB_PASSWORD="your-password" +``` + +--- + +## Code standards + +- **UK English** in comments, docstrings and user-facing strings (e.g. "analyse", "colour", "metres"). +- **Docstrings** use reStructuredText style (`:param`, `:type`, `:return`, `:rtype`) — see existing functions + for examples. +- **Follow existing patterns** in the codebase. +- **Wind and solar industry terminology** used correctly and consistently. + +--- + +The library is licensed under the MIT license. By contributing you agree that your contributions will be licensed +under the same terms. diff --git a/tests/test_load.py b/tests/test_load.py index eebf0eaf..7bafbb5c 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -160,6 +160,14 @@ def test_load_campbell_scientific(): data1['2016-01-09 15:30:00':'2016-01-10 23:50:00'].fillna(-999)).all().all() +@pytest.mark.skipif( + not ( + (os.environ.get('BRIGHTHUB_CLIENT_ID') and os.environ.get('BRIGHTHUB_CLIENT_SECRET')) + or (os.environ.get('BRIGHTHUB_EMAIL') and os.environ.get('BRIGHTHUB_PASSWORD')) + ), + reason="Either BRIGHTHUB_CLIENT_ID and BRIGHTHUB_CLIENT_SECRET, or BRIGHTHUB_EMAIL and BRIGHTHUB_PASSWORD " + "must be set to run this integration test against the live BrightHub API." +) def test_load_brighthub(): plant_uuid = '7a58497e-bee1-42a2-8084-c47a5cf213b7' @@ -263,7 +271,10 @@ def test_load_brighthub_get_data_csv_explicit_no_warning(): def test_load_brighthub_get_data_parquet(): - pytest.importorskip('pyarrow', reason="parquet engine required to round-trip parquet bytes") + try: + import pyarrow # noqa: F401 + except ImportError: + pytest.importorskip('fastparquet', reason="parquet engine (pyarrow or fastparquet) required to round-trip parquet bytes") index = pd.to_datetime(['2016-06-01 00:00:00', '2016-06-01 00:10:00']) index.name = 'Timestamp' From 2ea38edb8e99950cef6f20ad0358214fa431c708 Mon Sep 17 00:00:00 2001 From: dancasey-ie Date: Thu, 14 May 2026 16:09:53 +0100 Subject: [PATCH 4/5] iss609 support nested conditions and time_range_conditions in apply_cleaning_rules (#610) Updated apply_cleaning_rules() to support nested boolean conditions (and/or/not) inside the conditions block and a new optional time_range_conditions block on cleaning rules, in line with the updated BrightHub /measurement-locations/{uuid}/cleaning-rules API response. Existing flat-condition cleaning rule files continue to work unchanged. The new measurement_point_uuid and statistic_type_id fields on conditions and clean_out items are accepted but currently ignored. Extended cleaning_rule.schema.json with a time_condition definition. Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: stephenholleran --- CHANGELOG.md | 1 + brightwind/load/cleaning_rule.schema.json | 41 ++++- brightwind/load/load.py | 152 +++++++++++------- tests/test_load.py | 179 ++++++++++++++++++++++ 4 files changed, 318 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6245a5be..6fa14271 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Additional labels for pre-release and build metadata are available as extensions XX-May-2026 ### New Features and Enhancements +1. Updated `apply_cleaning_rules()` to support nested boolean conditions (`and`/`or`/`not`) inside the `conditions` block and a new optional `time_range_conditions` block on cleaning rules, in line with the updated BrightHub `/measurement-locations/{uuid}/cleaning-rules` API response. Existing flat-condition cleaning rule files continue to work unchanged. The new `measurement_point_uuid` and `statistic_type_id` fields on `conditions` and `clean_out` items are accepted but currently ignored — column resolution still relies on `assembled_column_name`. The `cleaning_rule.schema.json` was extended with a `time_condition` definition. 1. Rewrote `README.md` and `contributing.md` to align with the [docs site](https://brightwind-dev.github.io/brightwind-docs/), with clearer install options (venv / conda), a quick-start example, and an expanded contributing guide covering the fork workflow, editable dev install and running the test suite. ([#602](https://github.com/brightwind-dev/brightwind/issues/602)) 2. `test_load_brighthub` is now skipped automatically when no BrightHub credentials are available — either `BRIGHTHUB_CLIENT_ID` and `BRIGHTHUB_CLIENT_SECRET`, or the deprecated `BRIGHTHUB_EMAIL` and `BRIGHTHUB_PASSWORD` — so the test suite runs cleanly without BrightHub credentials. ([#602](https://github.com/brightwind-dev/brightwind/issues/602)) 1. Added support for reading Parquet files from BrightHub in `LoadBrightHub.get_data()` via a new `file_extension` argument (one of `'.csv'` or `'.parquet'`). Default remains `'.csv'` for backwards compatibility; Parquet is expected to become the default in the next major release. Reading Parquet requires a parquet engine — install with `pip install brightwind[parquet]` (pyarrow) or `pip install brightwind[parquet-fastparquet]` (fastparquet). ([#601](https://github.com/brightwind-dev/brightwind/issues/601)) diff --git a/brightwind/load/cleaning_rule.schema.json b/brightwind/load/cleaning_rule.schema.json index ca2280f7..35af98b1 100644 --- a/brightwind/load/cleaning_rule.schema.json +++ b/brightwind/load/cleaning_rule.schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$version": "1.0.0", + "$version": "1.1.0", "definitions": { "condition": { "type": "object", @@ -47,6 +47,44 @@ { "required": ["or"], "maxProperties": 1 }, { "required": ["not"], "maxProperties": 1 } ] + }, + "time_condition": { + "type": "object", + "properties": { + "value": { + "type": "string", + "format": "date-time", + "description": "An ISO 8601 timestamp compared against the timeseries datetime index." + }, + "comparison_operator_id": { + "type": "integer", + "oneOf": [ + { "const": 1, "description": "is less than" }, + { "const": 2, "description": "is less than or equal" }, + { "const": 3, "description": "is greater than" }, + { "const": 4, "description": "is greater than or equal" }, + { "const": 5, "description": "equals" }, + { "const": 6, "description": "not equals" } + ] + }, + "and": { + "type": "array", + "items": { "$ref": "#/definitions/time_condition" }, + "minItems": 2 + }, + "or": { + "type": "array", + "items": { "$ref": "#/definitions/time_condition" }, + "minItems": 2 + }, + "not": { "$ref": "#/definitions/time_condition" } + }, + "oneOf": [ + { "required": ["value", "comparison_operator_id"] }, + { "required": ["and"], "maxProperties": 1 }, + { "required": ["or"], "maxProperties": 1 }, + { "required": ["not"], "maxProperties": 1 } + ] } }, "type": "object", @@ -55,6 +93,7 @@ "type": "object", "properties": { "conditions": { "$ref": "#/definitions/condition" }, + "time_range_conditions": { "$ref": "#/definitions/time_condition" }, "clean_out": { "type": "array", "items": { diff --git a/brightwind/load/load.py b/brightwind/load/load.py index 965b1586..50b5ed5a 100644 --- a/brightwind/load/load.py +++ b/brightwind/load/load.py @@ -18,6 +18,7 @@ import concurrent import math import operator +import functools __all__ = ['load_csv', @@ -2515,53 +2516,60 @@ def apply_cleaning(data, cleaning_file_or_df, inplace=False, sensor_col_name='Se return data -def _apply_cleaning_rule(df, condition_col, target_cols, comparator_value, comparison_operator_id, replacement_value, - date_from, date_to): - """Apply cleaning rule based on a single column to replace values on a list of columns - - :param df: Data to be cleaned. - :type df: pandas.DataFrame - :param condition_col: Column that the cleaning rule is based on - :type condition_col: str - :param target_cols: Column to apply the cleaning rule to clean the data of - :type target_cols: List[str] - :param comparator_value: Threshold value to use in the cleaning rule, defined by - the cleaning_rule_json format. - :type comparator_value: float - :param comparison_operator_id: Operator code (1-6) defined by the cleaning_rule_json format, to be used with - variable operator_dict to define the operator used on the condition column. - Code number corresponds to the following operators: - 1: is less than (<), - 2: is less than or equal (≤), - 3: is greater than (>), - 4: is greater than or equal (≥), - 5: equals (=), - 6: not equals (≠) - :type comparison_operator_id: int - :param replacement_value: Value or string to replace the data in the target_cols with - :type replacement_value: str | np.nan - :param dates_from: List of datetimes that the cleaning rule should be applied from for each column. - :type dates_from: str - :param dates_to: List of datetimes that the cleaning rule should be applied until for each column. If - None is present the data is cleaned until the end of the file. - :type dates_to: str| None - :return: Cleaned data - :rtype: pandas.DataFrame +def _build_date_range_mask(df, date_from, date_to): + """Build a boolean Series selecting rows of df.index within [date_from, date_to). + + `date_from` is inclusive, `date_to` is exclusive (matches the schema documentation). + Either bound may be None — a None bound is treated as unbounded on that side. """ + if date_from is None and date_to is None: + return pd.Series(True, index=df.index) + if date_to is None: + return pd.Series(df.index >= pd.Timestamp(date_from), index=df.index) + if date_from is None: + return pd.Series(df.index < pd.Timestamp(date_to), index=df.index) + return pd.Series((df.index >= pd.Timestamp(date_from)) & (df.index < pd.Timestamp(date_to)), index=df.index) - result_df = df - op = OPERATOR_DICT[comparison_operator_id] - mask = op(df[condition_col], comparator_value) - if date_to: - date_filter = (df.index >= date_from) & (df.index < date_to) - else: - date_filter = (df.index >= date_from) - for col in target_cols: - mask_date_range = mask & date_filter - result_df.loc[mask_date_range, col] = replacement_value - - return result_df +def _evaluate_condition(df, condition): + """Recursively evaluate a column-based condition node and return a boolean Series aligned to df.index. + + A node is either a single comparison with ``assembled_column_name`` / ``comparison_operator_id`` / + ``comparator_value``, or a boolean combinator with one of ``and`` / ``or`` / ``not`` wrapping + further nodes. Schema validation is assumed to have already enforced exactly one shape per node. + """ + if 'and' in condition: + return functools.reduce(operator.and_, + (_evaluate_condition(df, child) for child in condition['and'])) + if 'or' in condition: + return functools.reduce(operator.or_, + (_evaluate_condition(df, child) for child in condition['or'])) + if 'not' in condition: + return ~_evaluate_condition(df, condition['not']) + + op = OPERATOR_DICT[condition['comparison_operator_id']] + return op(df[condition['assembled_column_name']], condition['comparator_value']) + + +def _evaluate_time_range_condition(df, condition): + """Recursively evaluate a time-range condition node and return a boolean Series aligned to df.index. + + Mirrors :func:`_evaluate_condition` but each single comparison compares ``df.index`` against + an ISO timestamp ``value`` rather than comparing a column against a numeric ``comparator_value``. + """ + if 'and' in condition: + return functools.reduce(operator.and_, + (_evaluate_time_range_condition(df, child) for child in condition['and'])) + if 'or' in condition: + return functools.reduce(operator.or_, + (_evaluate_time_range_condition(df, child) for child in condition['or'])) + if 'not' in condition: + return ~_evaluate_time_range_condition(df, condition['not']) + + op = OPERATOR_DICT[condition['comparison_operator_id']] + result = op(df.index.to_series(), pd.Timestamp(condition['value'])) + result.name = None + return result def apply_cleaning_rules(data, cleaning_rules_file_or_list, inplace=False, replacement_text='NaN'): @@ -2619,6 +2627,39 @@ def apply_cleaning_rules(data, cleaning_rules_file_or_list, inplace=False, repla schema = json.load(file) schema + **Nested conditions** + + The ``conditions`` block may be a single flat comparison or a recursive tree built with + ``and`` / ``or`` / ``not``. All forms are evaluated against the supplied DataFrame and the + resulting boolean mask is applied to the columns listed in ``clean_out``. + :: + "conditions": {"and": [ + {"assembled_column_name": "Spd80mN", "comparison_operator_id": 1, "comparator_value": 10}, + {"assembled_column_name": "T2m", "comparison_operator_id": 3, "comparator_value": 5} + ]} + + **Time-range conditions** + + An optional ``time_range_conditions`` block (same nested ``and`` / ``or`` / ``not`` structure, + with ISO-timestamp ``value`` leaves) restricts cleaning to specific time windows. It is ANDed + with ``conditions`` and any ``date_from`` / ``date_to``. + :: + "time_range_conditions": {"and": [ + {"value": "2016-02-01T00:00:00", "comparison_operator_id": 4}, + {"value": "2017-09-01T00:00:00", "comparison_operator_id": 1} + ]} + + **Stat-type expansion of ``clean_out``** + + A ``clean_out`` item naming the avg column will also clean all associated stat-type columns at + the same measurement point (``_sd``, ``_max``, ``_min``, ``_count``, ``_availability``, + ``_quality``, ``_sum``, ``_median``, ``_mode``, ``_range``, ``_gust``, ``_ti``, ``_ti30sec``, + ``_text``, ``_val``) via substring matching on column names. + + The new ``measurement_point_uuid`` and ``statistic_type_id`` fields on ``conditions`` and + ``clean_out`` items (added in the BrightHub API) are accepted but currently ignored — only + ``assembled_column_name`` is used for column resolution. + """ if inplace is False: @@ -2645,20 +2686,23 @@ def apply_cleaning_rules(data, cleaning_rules_file_or_list, inplace=False, repla if replacement_text == 'NaN': replacement_text = np.nan - + for cleaning_rule in cleaning_json: - columns_to_clean = [column_name['assembled_column_name'] for column_name in cleaning_rule['rule']['clean_out']] - date_from = cleaning_rule['rule'].get('date_from', data.index[0]) - date_to = cleaning_rule['rule'].get('date_to', None) - columns_to_clean = [column_name for column_to_clean in columns_to_clean for column_name in data.columns + rule = cleaning_rule['rule'] + + columns_to_clean = [c['assembled_column_name'] for c in rule['clean_out']] + columns_to_clean = [column_name for column_to_clean in columns_to_clean + for column_name in data.columns if column_to_clean in column_name] - condition_column_name = cleaning_rule['rule']['conditions']['assembled_column_name'] - comparator_value = cleaning_rule['rule']['conditions']['comparator_value'] - comparison_operator_id = cleaning_rule['rule']['conditions']['comparison_operator_id'] + mask = _evaluate_condition(data, rule['conditions']) + mask &= _build_date_range_mask(data, rule.get('date_from'), rule.get('date_to')) + if 'time_range_conditions' in rule: + mask &= _evaluate_time_range_condition(data, rule['time_range_conditions']) + + for col in columns_to_clean: + data.loc[mask, col] = replacement_text - data = _apply_cleaning_rule(data, condition_column_name, columns_to_clean, comparator_value, - comparison_operator_id, replacement_text, date_from, date_to) if inplace is False: return data diff --git a/tests/test_load.py b/tests/test_load.py index 7bafbb5c..b0615f0f 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -114,6 +114,185 @@ def test_apply_cleaning_rules(): assert (data[data_original["T2m"] > 5][date_from:date_to]["Spd80mS"] == "-").all() +def _synthetic_cleaning_df(): + """Deterministic synthetic DataFrame for nested-condition / time-range tests.""" + index = pd.date_range('2016-01-01', '2016-03-31 23:50', freq='10min') + n = len(index) + return pd.DataFrame( + { + 'Spd': np.linspace(0, 20, n), + 'Dir': np.linspace(0, 359, n), + 'T2m': np.linspace(-10, 30, n), + }, + index=index, + ) + + +def test_apply_cleaning_rules_nested_and_time_range(): + df = _synthetic_cleaning_df() + + # 1. Flat condition regression — equivalent to old single-condition behaviour. + rules = [{'rule': { + 'clean_out': [{'assembled_column_name': 'Spd'}], + 'conditions': {'assembled_column_name': 'Spd', 'comparison_operator_id': 1, 'comparator_value': 10}, + }}] + cleaned = bw.apply_cleaning_rules(df, rules) + expected_mask = df['Spd'] < 10 + assert cleaned.loc[expected_mask, 'Spd'].isna().all() + assert cleaned.loc[~expected_mask, 'Spd'].notna().all() + + # 2. AND of two single comparisons — both must be true. + rules = [{'rule': { + 'clean_out': [{'assembled_column_name': 'Spd'}], + 'conditions': {'and': [ + {'assembled_column_name': 'Spd', 'comparison_operator_id': 1, 'comparator_value': 10}, + {'assembled_column_name': 'T2m', 'comparison_operator_id': 1, 'comparator_value': 5}, + ]}, + }}] + cleaned = bw.apply_cleaning_rules(df, rules) + expected_mask = (df['Spd'] < 10) & (df['T2m'] < 5) + assert cleaned.loc[expected_mask, 'Spd'].isna().all() + assert cleaned.loc[~expected_mask, 'Spd'].notna().all() + + # 3. OR of two single comparisons — either may be true. + rules = [{'rule': { + 'clean_out': [{'assembled_column_name': 'Spd'}], + 'conditions': {'or': [ + {'assembled_column_name': 'Spd', 'comparison_operator_id': 1, 'comparator_value': 5}, + {'assembled_column_name': 'T2m', 'comparison_operator_id': 3, 'comparator_value': 20}, + ]}, + }}] + cleaned = bw.apply_cleaning_rules(df, rules) + expected_mask = (df['Spd'] < 5) | (df['T2m'] > 20) + assert cleaned.loc[expected_mask, 'Spd'].isna().all() + assert cleaned.loc[~expected_mask, 'Spd'].notna().all() + + # 4. NOT — negation of a single comparison. + rules = [{'rule': { + 'clean_out': [{'assembled_column_name': 'Spd'}], + 'conditions': {'not': {'assembled_column_name': 'Spd', 'comparison_operator_id': 1, 'comparator_value': 10}}, + }}] + cleaned = bw.apply_cleaning_rules(df, rules) + expected_mask = ~(df['Spd'] < 10) + assert cleaned.loc[expected_mask, 'Spd'].isna().all() + assert cleaned.loc[~expected_mask, 'Spd'].notna().all() + + # 5. Nested AND containing OR. + rules = [{'rule': { + 'clean_out': [{'assembled_column_name': 'Spd'}], + 'conditions': {'and': [ + {'assembled_column_name': 'Dir', 'comparison_operator_id': 4, 'comparator_value': 180}, + {'or': [ + {'assembled_column_name': 'Spd', 'comparison_operator_id': 1, 'comparator_value': 3}, + {'assembled_column_name': 'Spd', 'comparison_operator_id': 3, 'comparator_value': 15}, + ]}, + ]}, + }}] + cleaned = bw.apply_cleaning_rules(df, rules) + expected_mask = (df['Dir'] >= 180) & ((df['Spd'] < 3) | (df['Spd'] > 15)) + assert cleaned.loc[expected_mask, 'Spd'].isna().all() + assert cleaned.loc[~expected_mask, 'Spd'].notna().all() + + # 6. time_range_conditions alone (combined with an always-true conditions block). + rules = [{'rule': { + 'clean_out': [{'assembled_column_name': 'Spd'}], + 'conditions': {'assembled_column_name': 'Spd', 'comparison_operator_id': 4, 'comparator_value': -1}, + 'time_range_conditions': {'and': [ + {'value': '2016-02-01T00:00:00', 'comparison_operator_id': 4}, + {'value': '2016-03-01T00:00:00', 'comparison_operator_id': 1}, + ]}, + }}] + cleaned = bw.apply_cleaning_rules(df, rules) + expected_mask = (df.index >= pd.Timestamp('2016-02-01')) & (df.index < pd.Timestamp('2016-03-01')) + assert cleaned.loc[expected_mask, 'Spd'].isna().all() + assert cleaned.loc[~expected_mask, 'Spd'].notna().all() + + # 7. time_range_conditions ANDed with a measurement condition. + rules = [{'rule': { + 'clean_out': [{'assembled_column_name': 'Spd'}], + 'conditions': {'assembled_column_name': 'Spd', 'comparison_operator_id': 1, 'comparator_value': 10}, + 'time_range_conditions': {'and': [ + {'value': '2016-02-01T00:00:00', 'comparison_operator_id': 4}, + {'value': '2016-03-01T00:00:00', 'comparison_operator_id': 1}, + ]}, + }}] + cleaned = bw.apply_cleaning_rules(df, rules) + in_window = (df.index >= pd.Timestamp('2016-02-01')) & (df.index < pd.Timestamp('2016-03-01')) + expected_mask = (df['Spd'] < 10) & in_window + assert cleaned.loc[expected_mask, 'Spd'].isna().all() + assert cleaned.loc[~expected_mask, 'Spd'].notna().all() + + # 8. time_range_conditions combined with date_from/date_to — all three ANDed. + rules = [{'rule': { + 'clean_out': [{'assembled_column_name': 'Spd'}], + 'conditions': {'assembled_column_name': 'Spd', 'comparison_operator_id': 4, 'comparator_value': -1}, + 'date_from': '2016-01-15T00:00:00', + 'date_to': '2016-03-15T00:00:00', + 'time_range_conditions': {'and': [ + {'value': '2016-02-01T00:00:00', 'comparison_operator_id': 4}, + {'value': '2016-04-01T00:00:00', 'comparison_operator_id': 1}, + ]}, + }}] + cleaned = bw.apply_cleaning_rules(df, rules) + expected_mask = ( + (df.index >= pd.Timestamp('2016-01-15')) & (df.index < pd.Timestamp('2016-03-15')) & + (df.index >= pd.Timestamp('2016-02-01')) & (df.index < pd.Timestamp('2016-04-01')) + ) + assert cleaned.loc[expected_mask, 'Spd'].isna().all() + assert cleaned.loc[~expected_mask, 'Spd'].notna().all() + + # 9. Validation error — single comparison missing comparator_value. + rules = [{'rule': { + 'clean_out': [{'assembled_column_name': 'Spd'}], + 'conditions': {'assembled_column_name': 'Spd', 'comparison_operator_id': 1}, + }}] + with pytest.raises(ValueError) as except_info: + bw.apply_cleaning_rules(df, rules) + assert "validity of the supplied JSON" in str(except_info.value) + + # 10. Stat-type expansion — clean_out on the avg column also cleans all stat variants present. + index = pd.date_range('2016-01-01', '2016-01-02 23:50', freq='10min') + n = len(index) + df_stats = pd.DataFrame( + { + 'Spd_40m': np.linspace(0, 20, n), + 'Spd_40m_sd': np.linspace(0, 2, n), + 'Spd_40m_max': np.linspace(0, 25, n), + 'Spd_40m_min': np.linspace(0, 15, n), + 'OtherCol': np.linspace(0, 1, n), + }, + index=index, + ) + rules = [{'rule': { + 'clean_out': [{'assembled_column_name': 'Spd_40m'}], + 'conditions': {'assembled_column_name': 'Spd_40m', 'comparison_operator_id': 1, 'comparator_value': 10}, + }}] + cleaned = bw.apply_cleaning_rules(df_stats, rules) + mask = df_stats['Spd_40m'] < 10 + for col in ('Spd_40m', 'Spd_40m_sd', 'Spd_40m_max', 'Spd_40m_min'): + assert cleaned.loc[mask, col].isna().all() + assert cleaned.loc[~mask, col].notna().all() + assert cleaned['OtherCol'].notna().all() + + # New fields measurement_point_uuid and statistic_type_id are accepted but ignored. + rules = [{'rule': { + 'clean_out': [{ + 'assembled_column_name': 'Spd_40m', + 'measurement_point_uuid': '00000000-0000-0000-0000-000000000000', + 'statistic_type_id': 'avg', + }], + 'conditions': { + 'assembled_column_name': 'Spd_40m', + 'comparison_operator_id': 1, + 'comparator_value': 10, + 'measurement_point_uuid': '00000000-0000-0000-0000-000000000000', + 'statistic_type_id': 'avg', + }, + }}] + cleaned = bw.apply_cleaning_rules(df_stats, rules) + assert cleaned.loc[mask, 'Spd_40m'].isna().all() + + def test_load_csv(): data = bw.load_csv(os.path.join(DEMO_DATA_FOLDER, 'demo_data.csv')) data2 = bw.load_csv(os.path.join(DEMO_DATA_FOLDER, 'demo_data2.csv'), dayfirst=True) From 4ebe41069ca4b675927e011f283081a0925393fb Mon Sep 17 00:00:00 2001 From: stephenholleran Date: Thu, 14 May 2026 16:21:12 +0100 Subject: [PATCH 5/5] update version number to 2.7.0 --- CHANGELOG.md | 4 ++-- brightwind/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fa14271..bc9c8090 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,10 @@ Additional labels for pre-release and build metadata are available as extensions --- ## [2.7.0] -XX-May-2026 +14-May-2026 ### New Features and Enhancements -1. Updated `apply_cleaning_rules()` to support nested boolean conditions (`and`/`or`/`not`) inside the `conditions` block and a new optional `time_range_conditions` block on cleaning rules, in line with the updated BrightHub `/measurement-locations/{uuid}/cleaning-rules` API response. Existing flat-condition cleaning rule files continue to work unchanged. The new `measurement_point_uuid` and `statistic_type_id` fields on `conditions` and `clean_out` items are accepted but currently ignored — column resolution still relies on `assembled_column_name`. The `cleaning_rule.schema.json` was extended with a `time_condition` definition. +1. Updated `apply_cleaning_rules()` to support nested boolean conditions (`and`/`or`/`not`) inside the `conditions` block and a new optional `time_range_conditions` block on cleaning rules, in line with the updated BrightHub `/measurement-locations/{uuid}/cleaning-rules` API response. Existing flat-condition cleaning rule files continue to work unchanged. The new `measurement_point_uuid` and `statistic_type_id` fields on `conditions` and `clean_out` items are accepted but currently ignored — column resolution still relies on `assembled_column_name`. The `cleaning_rule.schema.json` was extended with a `time_condition` definition. ([#609](https://github.com/brightwind-dev/brightwind/issues/609)) 1. Rewrote `README.md` and `contributing.md` to align with the [docs site](https://brightwind-dev.github.io/brightwind-docs/), with clearer install options (venv / conda), a quick-start example, and an expanded contributing guide covering the fork workflow, editable dev install and running the test suite. ([#602](https://github.com/brightwind-dev/brightwind/issues/602)) 2. `test_load_brighthub` is now skipped automatically when no BrightHub credentials are available — either `BRIGHTHUB_CLIENT_ID` and `BRIGHTHUB_CLIENT_SECRET`, or the deprecated `BRIGHTHUB_EMAIL` and `BRIGHTHUB_PASSWORD` — so the test suite runs cleanly without BrightHub credentials. ([#602](https://github.com/brightwind-dev/brightwind/issues/602)) 1. Added support for reading Parquet files from BrightHub in `LoadBrightHub.get_data()` via a new `file_extension` argument (one of `'.csv'` or `'.parquet'`). Default remains `'.csv'` for backwards compatibility; Parquet is expected to become the default in the next major release. Reading Parquet requires a parquet engine — install with `pip install brightwind[parquet]` (pyarrow) or `pip install brightwind[parquet-fastparquet]` (fastparquet). ([#601](https://github.com/brightwind-dev/brightwind/issues/601)) diff --git a/brightwind/__init__.py b/brightwind/__init__.py index 7c43d7f9..13324470 100644 --- a/brightwind/__init__.py +++ b/brightwind/__init__.py @@ -34,4 +34,4 @@ __all__ = ['analyse', 'transform', 'export', 'load', 'demo_datasets'] -__version__ = '2.7.0_dev' +__version__ = '2.7.0' diff --git a/setup.py b/setup.py index 5f814370..2671ec73 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ def get_version(rel_path): }, url='https://github.com/brightwind-dev/brightwind.git', # UPDATE VERSION NUMBER HERE: - download_url='https://github.com/brightwind-dev/brightwind/archive/v2.6.0.tar.gz', + download_url='https://github.com/brightwind-dev/brightwind/archive/v2.7.0.tar.gz', license='MIT', author='Stephen Holleran of BrightWind Ltd', author_email='stephen@brightwindanalysis.com',