From ad24993b2379eb861ae8ea7ef964d07eaa5a122d Mon Sep 17 00:00:00 2001 From: James McCreight Date: Tue, 12 Aug 2025 23:58:37 -0600 Subject: [PATCH 1/3] update asv.conf.json --- asv_benchmarks/asv.conf.json | 380 +++++++++++++++++------------------ 1 file changed, 190 insertions(+), 190 deletions(-) diff --git a/asv_benchmarks/asv.conf.json b/asv_benchmarks/asv.conf.json index ada26271..d8c8ff49 100644 --- a/asv_benchmarks/asv.conf.json +++ b/asv_benchmarks/asv.conf.json @@ -1,192 +1,192 @@ { - // The version of the config file format. Do not change, unless - // you know what you are doing. - "version": 1, - - // The name of the project being benchmarked - "project": "pywatershed", - - // The project's homepage - "project_url": "http://github.com/EC-USGS/pywatershed", - - // The URL or local path of the source code repository for the - // project being benchmarked - "repo": "..", - - // The Python project's subdirectory in your repo. If missing or - // the empty string, the project is assumed to be located at the root - // of the repository. - // "repo_subdir": "", - - // Customizable commands for building, installing, and - // uninstalling the project. See asv.conf.json documentation. - // - // "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], - // "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], - // "build_command": [ - // "python setup.py build", - // "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}" - // ], - - // List of branches to benchmark. If not provided, defaults to "master" - // (for git) or "default" (for mercurial). - // "branches": ["main"], // for git - "branches": ["develop"], // for git - // "branches": ["default"], // for mercurial - - // The DVCS being used. If not set, it will be automatically - // determined from "repo" by looking at the protocol in the URL - // (if remote), or by looking for special directories, such as - // ".git" (if local). - "dvcs": "git", - - // The tool to use to create environments. May be "conda", - // "virtualenv" or other value depending on the plugins in use. - // If missing or the empty string, the tool will be automatically - // determined by looking for tools on the PATH environment - // variable. - "environment_type": "conda", - - // timeout in seconds for installing any dependencies in environment - // defaults to 10 min - "install_timeout": 600, - - // the base URL to show a commit for the project. - "show_commit_url": "http://github.com/EC-USGS/pywatershed/commit/", - - // The Pythons you'd like to test against. If not provided, defaults - // to the current version of Python used to run `asv`. - "pythons": ["3.10"], - - // The list of conda channel names to be searched for benchmark - // dependency packages in the specified order - "conda_channels": ["conda-forge", "defaults"], - - // A conda environment file that is used for environment creation. - "conda_environment_file": "../environment.yml", - - // The matrix of dependencies to test. Each key of the "req" - // requirements dictionary is the name of a package (in PyPI) and - // the values are version numbers. An empty list or empty string - // indicates to just test against the default (latest) - // version. null indicates that the package is to not be - // installed. If the package to be tested is only available from - // PyPi, and the 'environment_type' is conda, then you can preface - // the package name by 'pip+', and the package will be installed - // via pip (with all the conda available packages installed first, - // followed by the pip installed packages). - // - // The ``@env`` and ``@env_nobuild`` keys contain the matrix of - // environment variables to pass to build and benchmark commands. - // An environment will be created for every combination of the - // cartesian product of the "@env" variables in this matrix. - // Variables in "@env_nobuild" will be passed to every environment - // during the benchmark phase, but will not trigger creation of - // new environments. A value of ``null`` means that the variable - // will not be set for the current combination. - // - // "matrix": { - // "req": { - // "numpy": ["1.6", "1.7"], - // "six": ["", null], // test with and without six installed - // "pip+emcee": [""] // emcee is only available for install with pip. - // }, - // "env": {"ENV_VAR_1": ["val1", "val2"]}, - // "env_nobuild": {"ENV_VAR_2": ["val3", null]}, - // }, - - "matrix": { - // "setuptools_scm": [""], // GH6609 - "numpy": [""], - "netcdf4": [""], - "xarray": ["",] - }, - - // Combinations of libraries/python versions can be excluded/included - // from the set to test. Each entry is a dictionary containing additional - // key-value pairs to include/exclude. - // - // An exclude entry excludes entries where all values match. The - // values are regexps that should match the whole string. - // - // An include entry adds an environment. Only the packages listed - // are installed. The 'python' key is required. The exclude rules - // do not apply to includes. - // - // In addition to package names, the following keys are available: - // - // - python - // Python version, as in the *pythons* variable above. - // - environment_type - // Environment type, as above. - // - sys_platform - // Platform, as in sys.platform. Possible values for the common - // cases: 'linux2', 'win32', 'cygwin', 'darwin'. - // - req - // Required packages - // - env - // Environment variables - // - env_nobuild - // Non-build environment variables - // - // "exclude": [ - // {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows - // {"environment_type": "conda", "req": {"six": null}}, // don't run without six on conda - // {"env": {"ENV_VAR_1": "val2"}}, // skip val2 for ENV_VAR_1 - // ], - // - // "include": [ - // // additional env for python2.7 - // {"python": "2.7", "req": {"numpy": "1.8"}, "env_nobuild": {"FOO": "123"}}, - // // additional env if run on windows+conda - // {"platform": "win32", "environment_type": "conda", "python": "2.7", "req": {"libpython": ""}}, - // ], - - // The directory (relative to the current directory) that benchmarks are - // stored in. If not provided, defaults to "benchmarks" - "benchmark_dir": "benchmarks", - - // The directory (relative to the current directory) to cache the Python - // environments in. If not provided, defaults to "env" - "env_dir": ".asv/env", - - // The directory (relative to the current directory) that raw benchmark - // results are stored in. If not provided, defaults to "results". - "results_dir": ".asv/results", - - // The directory (relative to the current directory) that the html tree - // should be written to. If not provided, defaults to "html". - "html_dir": ".asv/html", - - // The number of characters to retain in the commit hashes. - // "hash_length": 8, - - // `asv` will cache results of the recent builds in each - // environment, making them faster to install next time. This is - // the number of builds to keep, per environment. - // "build_cache_size": 2, - - // The commits after which the regression search in `asv publish` - // should start looking for regressions. Dictionary whose keys are - // regexps matching to benchmark names, and values corresponding to - // the commit (exclusive) after which to start looking for - // regressions. The default is to start from the first commit - // with results. If the commit is `null`, regression detection is - // skipped for the matching benchmark. - // - // "regressions_first_commits": { - // "some_benchmark": "352cdf", // Consider regressions only after this commit - // "another_benchmark": null, // Skip regression detection altogether - // }, - - // The thresholds for relative change in results, after which `asv - // publish` starts reporting regressions. Dictionary of the same - // form as in ``regressions_first_commits``, with values - // indicating the thresholds. If multiple entries match, the - // maximum is taken. If no entry matches, the default is 5%. - // - // "regressions_thresholds": { - // "some_benchmark": 0.01, // Threshold of 1% - // "another_benchmark": 0.5, // Threshold of 50% - // }, + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "pywatershed", + + // The project's homepage + "project_url": "http://github.com/EC-USGS/pywatershed", + + // The URL or local path of the source code repository for the + // project being benchmarked + "repo": "..", + + // The Python project's subdirectory in your repo. If missing or + // the empty string, the project is assumed to be located at the root + // of the repository. + // "repo_subdir": "", + + // Customizable commands for building, installing, and + // uninstalling the project. See asv.conf.json documentation. + // + // "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], + // "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], + // "build_command": [ + // "python setup.py build", + // "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}" + // ], + + // List of branches to benchmark. If not provided, defaults to "master" + // (for git) or "default" (for mercurial). + // "branches": ["main"], // for git + "branches": ["develop"], // for git + // "branches": ["default"], // for mercurial + + // The DVCS being used. If not set, it will be automatically + // determined from "repo" by looking at the protocol in the URL + // (if remote), or by looking for special directories, such as + // ".git" (if local). + "dvcs": "git", + + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "conda", + + // timeout in seconds for installing any dependencies in environment + // defaults to 10 min + "install_timeout": 600, + + // the base URL to show a commit for the project. + "show_commit_url": "http://github.com/EC-USGS/pywatershed/commit/", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + "pythons": ["3.11"], + + // The list of conda channel names to be searched for benchmark + // dependency packages in the specified order + "conda_channels": ["conda-forge", "defaults"], + + // A conda environment file that is used for environment creation. + "conda_environment_file": "../environment.yml", + + // The matrix of dependencies to test. Each key of the "req" + // requirements dictionary is the name of a package (in PyPI) and + // the values are version numbers. An empty list or empty string + // indicates to just test against the default (latest) + // version. null indicates that the package is to not be + // installed. If the package to be tested is only available from + // PyPi, and the 'environment_type' is conda, then you can preface + // the package name by 'pip+', and the package will be installed + // via pip (with all the conda available packages installed first, + // followed by the pip installed packages). + // + // The ``@env`` and ``@env_nobuild`` keys contain the matrix of + // environment variables to pass to build and benchmark commands. + // An environment will be created for every combination of the + // cartesian product of the "@env" variables in this matrix. + // Variables in "@env_nobuild" will be passed to every environment + // during the benchmark phase, but will not trigger creation of + // new environments. A value of ``null`` means that the variable + // will not be set for the current combination. + // + // "matrix": { + // "req": { + // "numpy": ["1.6", "1.7"], + // "six": ["", null], // test with and without six installed + // "pip+emcee": [""] // emcee is only available for install with pip. + // }, + // "env": {"ENV_VAR_1": ["val1", "val2"]}, + // "env_nobuild": {"ENV_VAR_2": ["val3", null]}, + // }, + + "matrix": { + // "setuptools_scm": [""], // GH6609 + // "numpy": [""], + // "netcdf4": [""], + // "xarray": [""] + }, + + // Combinations of libraries/python versions can be excluded/included + // from the set to test. Each entry is a dictionary containing additional + // key-value pairs to include/exclude. + // + // An exclude entry excludes entries where all values match. The + // values are regexps that should match the whole string. + // + // An include entry adds an environment. Only the packages listed + // are installed. The 'python' key is required. The exclude rules + // do not apply to includes. + // + // In addition to package names, the following keys are available: + // + // - python + // Python version, as in the *pythons* variable above. + // - environment_type + // Environment type, as above. + // - sys_platform + // Platform, as in sys.platform. Possible values for the common + // cases: 'linux2', 'win32', 'cygwin', 'darwin'. + // - req + // Required packages + // - env + // Environment variables + // - env_nobuild + // Non-build environment variables + // + // "exclude": [ + // {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows + // {"environment_type": "conda", "req": {"six": null}}, // don't run without six on conda + // {"env": {"ENV_VAR_1": "val2"}}, // skip val2 for ENV_VAR_1 + // ], + // + // "include": [ + // // additional env for python2.7 + // {"python": "2.7", "req": {"numpy": "1.8"}, "env_nobuild": {"FOO": "123"}}, + // // additional env if run on windows+conda + // {"platform": "win32", "environment_type": "conda", "python": "2.7", "req": {"libpython": ""}}, + // ], + + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + "benchmark_dir": "benchmarks", + + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + "env_dir": ".asv/env", + + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + "results_dir": ".asv/results", + + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + "html_dir": ".asv/html" + + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + + // `asv` will cache results of the recent builds in each + // environment, making them faster to install next time. This is + // the number of builds to keep, per environment. + // "build_cache_size": 2, + + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + // "regressions_first_commits": { + // "some_benchmark": "352cdf", // Consider regressions only after this commit + // "another_benchmark": null, // Skip regression detection altogether + // }, + + // The thresholds for relative change in results, after which `asv + // publish` starts reporting regressions. Dictionary of the same + // form as in ``regressions_first_commits``, with values + // indicating the thresholds. If multiple entries match, the + // maximum is taken. If no entry matches, the default is 5%. + // + // "regressions_thresholds": { + // "some_benchmark": 0.01, // Threshold of 1% + // "another_benchmark": 0.5, // Threshold of 50% + // }, } From 6f45531c388201cdb342b80ea7f34d5bffa5b70e Mon Sep 17 00:00:00 2001 From: James McCreight Date: Wed, 13 Aug 2025 00:19:04 -0600 Subject: [PATCH 2/3] some asv fixes --- asv_benchmarks/asv.conf.json | 8 ++------ pywatershed/hydrology/prms_channel_flow_graph.py | 4 ++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/asv_benchmarks/asv.conf.json b/asv_benchmarks/asv.conf.json index d8c8ff49..74a2fb72 100644 --- a/asv_benchmarks/asv.conf.json +++ b/asv_benchmarks/asv.conf.json @@ -21,12 +21,8 @@ // Customizable commands for building, installing, and // uninstalling the project. See asv.conf.json documentation. // - // "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], - // "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], - // "build_command": [ - // "python setup.py build", - // "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}" - // ], + "install_command": ["python -mpip install ."], + "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], // List of branches to benchmark. If not provided, defaults to "master" // (for git) or "default" (for mercurial). diff --git a/pywatershed/hydrology/prms_channel_flow_graph.py b/pywatershed/hydrology/prms_channel_flow_graph.py index 7d0df6db..6a6eead9 100644 --- a/pywatershed/hydrology/prms_channel_flow_graph.py +++ b/pywatershed/hydrology/prms_channel_flow_graph.py @@ -383,8 +383,8 @@ def _calculate_subtimestep_numpy( ) -numba_msg = "prms_channel_flow_graph jit compiling with numba" -print(numba_msg, flush=True) +# numba_msg = "prms_channel_flow_graph jit compiling with numba" +# print(numba_msg, flush=True) _calculate_subtimestep_numba = nb.njit( nb.types.UniTuple(nb.float64, 5)( From 3386eec3f6d08b8767f8c22b5ac04132ce244336 Mon Sep 17 00:00:00 2001 From: James McCreight Date: Tue, 20 Jan 2026 09:26:52 -0700 Subject: [PATCH 3/3] single white space --- DEVELOPER.md | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/DEVELOPER.md b/DEVELOPER.md index f9e0d0fe..e9d47850 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -46,9 +46,9 @@ installation process varies widely. We are looking for a conda-based approach to obtaining compilers, but currently do not have a solution. Compilers are needed for two applications: - 1. Compiling and running C/Fortran PRMS code to generate testing/verification - data 2. Compiling (installing) and running fortran backends/kernels for some - hydrological process representations in pywatershed +1. Compiling and running C/Fortran PRMS code to generate testing/verification + data 2. Compiling (installing) and running fortran backends/kernels for some + hydrological process representations in pywatershed On Apple Silicon, the PRMS source code is only currently known to compile with intel while the fortran kernels in pywatershed only compile with gnu. @@ -83,7 +83,7 @@ To install all dependencies with `pip`: pip install ".[all]" ``` -Several dependency groups are defined in `pyproject.toml` and can be selected +Several dependency groups are defined in `pyproject.toml` and can be selected instead of `all` for a more lightweight environment: - `lint` @@ -99,8 +99,7 @@ in the root, `source venv/bin/activate`), `pywatershed` can be installed in mode](https://setuptools.pypa.io/en/latest/userguide/development_mode.html) with: -``` pip install -e . ``` - +`pip install -e . ` #### F2PY @@ -110,7 +109,7 @@ numpy](https://numpy.org/doc/stable/f2py/index.html). This repository is configured NOT to compile on install by default. Currently, we have not established this compilation procedure for Windows. On Linux and MacOS, compilation of fortran kernels on package installation is achieved by setting -several environent variables before installing the `pywatershed` module. For +several environent variables before installing the `pywatershed` module. For instance, from the project root: ``` @@ -123,8 +122,8 @@ pip install -e . Note that an editable (`-e` above) is required to compile the fotran code. - ## Branching model + This project uses the [git flow](https://nvie.com/posts/a-successful-git-branching-model/): development occurs on the `develop` branch, while `main` is reserved for the state of the @@ -132,14 +131,14 @@ latest release. Development PRs are typically squashed to `develop`, to avoid merge commits. At release time, release branches are merged to `main`, and then `main` is merged back into `develop`. - ## CI + The automated practices of installing, linting, and testing described below are all formally encoded in `.github/workflows/ci.yaml` and `.github/workflows/ci_examples.yaml` files. - ## Testing + Once the dependencies are available, we want to verify the software by running its test suite. However, we first need to generate the test data. This consists of running binaries (PRMS) and then converting the output to netcdf files used @@ -165,9 +164,10 @@ machine. For more details on the autotests, see [`autotest/README.md`](autotest/README.md). - ## Linting + Automated linting procedures are performed in CI and enforced, these are + ```shell ruff check . ruff format . @@ -175,12 +175,12 @@ ruff format . And you'll need to run these locally to pass CI checks. - ## Committing Jupyter Notebooks + All outputs are required to be stripped from jupyter notebooks prior to committing. To facilitate this we have [pre-commit hooks](https://pre-commit.com/) which will strip -outputs and metadata from jupyter notebooks. When a `git commit` is attempted, +outputs and metadata from jupyter notebooks. When a `git commit` is attempted, the hook will check all staged `*.ipynb` files. If the file is modified after running the hook (which runs [nbstripout](https://github.com/kynan/nbstripout)), then the @@ -194,6 +194,7 @@ time to keep very large diffs out of the repository history. If you are using The maximal amount of metadata can be stripped from Jupyter notebooks by following the example configuration found in the [nbstripout section on stripping metadata](https://github.com/kynan/nbstripout#stripping-metadata). ## pre-commit hooks + Pre-commit hooks apply actionas at commit-time. These are available when `pre-commit` is installed in the environment, as in the `environment.yml` supplied. To install yourself @@ -205,24 +206,21 @@ pre-commit install As specified in `.pre-commit-config.yaml`, we adopt the following pre-commit hooks -* [nbstripout](https://github.com/kynan/nbstripout): +- [nbstripout](https://github.com/kynan/nbstripout): strip outputs from jupyter notebooks -* [blackdoc](https://github.com/keewis/blackdoc): +- [blackdoc](https://github.com/keewis/blackdoc): apply black within documentation -* [doctoc](https://github.com/thlorenz/doctoc): auto generate tables of +- [doctoc](https://github.com/thlorenz/doctoc): auto generate tables of contents in markdown docs - ## Documentation + [Google-style docstrings](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) are used for documenting source code. (Though numpy style is supposedly handled as well by Napolean, preference is for google-style.) - - ## Miscellaneous - ### Locating the root Python scripts often need to reference files elsewhere in the project. To allow @@ -233,4 +231,4 @@ for file access, rather than using relative paths (e.g., `../some/path`). For a script in a subdirectory of the root, for instance, the conventional approach would be: -```Python project_root_path = Path(__file__).parent.parent ``` +`Python project_root_path = Path(__file__).parent.parent `