diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7528b6fc..be1ffbf4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -299,6 +299,13 @@ jobs: pip -V pip list + - name: Compile PRMS 5.2.1 + run: | + which gfortran + gfortran --version + which make + python -c "import pywatershed as pws; pws.utils.compile_prms('5.2.1')" + - name: sagehen_5yr_no_cascades - generate and manage test data domain, run PRMS and convert csv output to NetCDF working-directory: autotest run: | @@ -424,6 +431,13 @@ jobs: pip -V pip list + - name: Compile PRMS 5.2.1 + run: | + which gfortran + gfortran --version + which make + python -c "import pywatershed as pws; pws.utils.compile_prms('5.2.1')" + - name: hru_1 all configs - generate and manage test data, run PRMS and convert csv output to NetCDF working-directory: autotest run: | @@ -604,22 +618,13 @@ jobs: run: | python pywatershed/utils/gis_files.py - - name: Compile PRMS 5.2.1.1 - working-directory: prms_src/prms5.2.1.1 + - name: Compile PRMS 5.2.1 and 5.2.1.1 run: | which gfortran gfortran --version which make - MAKE_PATH=$(which make) - $MAKE_PATH clean MAKE="$MAKE_PATH" - $MAKE_PATH DBL_PREC=true FC=gfortran CC=gcc MAKE="$MAKE_PATH" - if [ "$RUNNER_OS" == "Windows" ]; then - cp bin/prms ../../bin/prms_5.2.1.1_gfort_win_dbl_prec.exe - elif [ "$RUNNER_OS" == "macOS" ]; then - cp bin/prms ../../bin/prms_5.2.1.1_gfortran_apple_silicon_dbl_prec - elif [ "$RUNNER_OS" == "Linux" ]; then - cp bin/prms ../../bin/prms_5.2.1.1_gfort_linux_dbl_prec - fi + python -c "import pywatershed as pws; pws.utils.compile_prms('5.2.1')" + python -c "import pywatershed as pws; pws.utils.compile_prms('5.2.1.1')" - name: drb_2yr all domains - generate and manage test data working-directory: autotest @@ -824,6 +829,13 @@ jobs: pip -V pip list + - name: Compile PRMS 5.2.1 + run: | + which gfortran + gfortran --version + which make + python -c "import pywatershed as pws; pws.utils.compile_prms('5.2.1')" + - name: ucb_2yr_nhm - generate and manage test data working-directory: autotest run: | @@ -940,6 +952,13 @@ jobs: pip -V pip list + - name: Compile PRMS 5.2.1 + run: | + which gfortran + gfortran --version + which make + python -c "import pywatershed as pws; pws.utils.compile_prms('5.2.1')" + - name: ucb_2yr_slow_tests - generate and manage test data working-directory: autotest run: | diff --git a/.gitignore b/.gitignore index 2f6e6568..b97b2e8b 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,11 @@ bin/libmf6.dylib bin/mf5to6 bin/mf6 bin/zbud6 +# PRMS binaries are compiled from prms_src/ on demand rather than checked +# in; only the GSFLOW binaries (whose source is not in this repo) are +# tracked in bin/. +bin/prms_* +bin/.compile_prms_*.lock # prms output files *.out @@ -116,3 +121,5 @@ starfit_minimal/ # claude code: personal settings stay untracked .claude/settings.local.json + +.codegraph/ diff --git a/DEVELOPER.md b/DEVELOPER.md index 8fc671c9..f54213a9 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -40,19 +40,23 @@ is a good source of information. ### Compilers -C and Fortran compilers are required. We are currently using gnu (gcc, -gfortran) 11 and 12 as well as intel (icc, ifort) 2021 on Windows, Linux, and -MacOS (including Apple Silicon). Both of these are freely obtainable but the -installation process varies widely. We are looking for a conda-based approach -to obtaining compilers, but currently do not have a solution. Compilers are -needed for two applications: - - 1. Compiling and running C/Fortran PRMS code to generate testing/verification - data 2. Compiling (installing) and running fortran backends/kernels for some - hydrological process representations in pywatershed - -On Apple Silicon, the PRMS source code is only currently known to compile with -intel while the fortran kernels in pywatershed only compile with gnu. +C and Fortran compilers are required. We use gnu (gcc, gfortran) +exclusively, on Windows, Linux, and Apple Silicon MacOS. The mamba/conda +environment described below supplies them (`gfortran>=15.2.0` in +`environment.yml`), so no separate compiler installation is needed. +Compilers are needed for compiling and running C/Fortran PRMS code to generate testing/verification data. + +The PRMS binaries used to generate test data are not kept in the +repository. They are compiled from `prms_src/` the first time they are +needed, by `pywatershed.utils.compile_prms()`, and written to `bin/` +(which is gitignored for these). `autotest/ci_local.sh` and `ci.yaml` call +the same function, so there is one build path everywhere. + +Intel compilers (icc, ifort) are no longer used or supported, and Intel +MacOS is no longer detected. The one remaining intel-built artifact is +`bin/gsflow_2.4.0_ifort_apple_silicon_dbl_prec`: GSFLOW source is not part +of this repository, so that binary is checked in rather than compiled. It +is an x86_64 build and therefore needs Rosetta 2 on Apple Silicon. ### Python @@ -84,7 +88,7 @@ To install all dependencies with `pip`: pip install ".[all]" ``` -Several dependency groups are defined in `pyproject.toml` and can be selected +Several dependency groups are defined in `pyproject.toml` and can be selected instead of `all` for a more lightweight environment: - `lint` @@ -100,8 +104,7 @@ in the root, `source venv/bin/activate`), `pywatershed` can be installed in mode](https://setuptools.pypa.io/en/latest/userguide/development_mode.html) with: -``` pip install -e . ``` - +`pip install -e . ` #### F2PY @@ -111,7 +114,7 @@ numpy](https://numpy.org/doc/stable/f2py/index.html). This repository is configured NOT to compile on install by default. Currently, we have not established this compilation procedure for Windows. On Linux and MacOS, compilation of fortran kernels on package installation is achieved by setting -several environent variables before installing the `pywatershed` module. For +several environent variables before installing the `pywatershed` module. For instance, from the project root: ``` @@ -124,8 +127,8 @@ pip install -e . Note that an editable (`-e` above) is required to compile the fotran code. - ## Branching model + This project uses the [git flow](https://nvie.com/posts/a-successful-git-branching-model/): development occurs on the `develop` branch, while `main` is reserved for the state of the @@ -133,16 +136,16 @@ latest release. Development PRs are typically squashed to `develop`, to avoid merge commits. At release time, release branches are merged to `main`, and then `main` is merged back into `develop`. - ## Maintenance ledger + Maintenance todos that are blocked on external events (dependency releases, cross-repo work like conda-forge feedstocks) are tracked in [`MAINTENANCE.md`](MAINTENANCE.md) at the repo root, with mechanically checkable unblock conditions. The `/maintenance` Claude skill checks them live and reports what is actionable. - ## CI + The automated practices of installing, linting, and testing described below are all formally encoded in `.github/workflows/ci.yaml` and `.github/workflows/ci_examples.yaml` files. @@ -166,7 +169,7 @@ The two variants complement each other: with no memory of earlier pushes: a push whose head commit message contains `ci-drb` runs the drb job for that push only, even on a branch previously pushed without any token, and later pushes without a token drop back to the - skeleton. Only the *head* (most recent) commit of a push is checked — when + skeleton. Only the _head_ (most recent) commit of a push is checked — when pushing several commits at once, the token must be in the last one. To trigger a domain run on the current state without changing any code, push an empty commit: @@ -185,7 +188,7 @@ lists, separators, or suffixes. Consequences: - Tokens can only add jobs, never subtract them: appending to a token (`ci-sagehen_gridded`) does not narrow the selection, it matches the `ci-sagehen` token and triggers every job in that family. -- A commit message that merely *mentions* a token triggers it — easy to do +- A commit message that merely _mentions_ a token triggers it — easy to do accidentally in a commit message about the CI configuration itself. Write `ci-` (as in this file) rather than a literal token when referring to the mechanism. @@ -204,8 +207,8 @@ Notes for maintaining the gates in `ci.yaml`: tokens, e.g. `ci-sagehen-5yr`, `ci-sagehen-gridded`, and `ci-sagehen-all` for the whole family. - ## Testing + Once the dependencies are available, we want to verify the software by running its test suite. However, we first need to generate the test data. This consists of running binaries (PRMS) and then converting the output to netcdf files used @@ -231,9 +234,10 @@ machine. For more details on the autotests, see [`autotest/README.md`](autotest/README.md). - ## Linting + Automated linting procedures are performed in CI and enforced, these are + ```shell ruff check . ruff format . @@ -241,12 +245,12 @@ ruff format . And you'll need to run these locally to pass CI checks. - ## Committing Jupyter Notebooks + All outputs are required to be stripped from jupyter notebooks prior to committing. To facilitate this we have [pre-commit hooks](https://pre-commit.com/) which will strip -outputs and metadata from jupyter notebooks. When a `git commit` is attempted, +outputs and metadata from jupyter notebooks. When a `git commit` is attempted, the hook will check all staged `*.ipynb` files. If the file is modified after running the hook (which runs [nbstripout](https://github.com/kynan/nbstripout)), then the @@ -260,6 +264,7 @@ time to keep very large diffs out of the repository history. If you are using The maximal amount of metadata can be stripped from Jupyter notebooks by following the example configuration found in the [nbstripout section on stripping metadata](https://github.com/kynan/nbstripout#stripping-metadata). ## pre-commit hooks + Pre-commit hooks apply actionas at commit-time. These are available when `pre-commit` is installed in the environment, as in the `environment.yml` supplied. To install yourself @@ -271,24 +276,21 @@ pre-commit install As specified in `.pre-commit-config.yaml`, we adopt the following pre-commit hooks -* [nbstripout](https://github.com/kynan/nbstripout): +- [nbstripout](https://github.com/kynan/nbstripout): strip outputs from jupyter notebooks -* [blackdoc](https://github.com/keewis/blackdoc): +- [blackdoc](https://github.com/keewis/blackdoc): apply black within documentation -* [doctoc](https://github.com/thlorenz/doctoc): auto generate tables of +- [doctoc](https://github.com/thlorenz/doctoc): auto generate tables of contents in markdown docs - ## Documentation + [Google-style docstrings](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) are used for documenting source code. (Though numpy style is supposedly handled as well by Napolean, preference is for google-style.) - - ## Miscellaneous - ### Locating the root Python scripts often need to reference files elsewhere in the project. To allow @@ -299,4 +301,4 @@ for file access, rather than using relative paths (e.g., `../some/path`). For a script in a subdirectory of the root, for instance, the conventional approach would be: -```Python project_root_path = Path(__file__).parent.parent ``` +`Python project_root_path = Path(__file__).parent.parent ` diff --git a/MAINTENANCE.md b/MAINTENANCE.md index d9b9f2a9..687c1810 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -4,9 +4,9 @@ - [Maintenance ledger](#maintenance-ledger) - [Open](#open) - [3.0.1 hotfix release (flopy pin unwind + pyPRMS floor)](#301-hotfix-release-flopy-pin-unwind--pyprms-floor) - - [conda-forge feedstock: pyprms floor replaces packaging pin](#conda-forge-feedstock-pyprms-floor-replaces-packaging-pin) - [pyPRMS upstream: close PR #64, port its regression test](#pyprms-upstream-close-pr-64-port-its-regression-test) - [Reconcile check_version.yaml with release_preflight.sh](#reconcile-check_versionyaml-with-release_preflightsh) + - [Retire the GSFLOW ifort binary](#retire-the-gsflow-ifort-binary) - [Deliver CI-usage findings to org admins](#deliver-ci-usage-findings-to-org-admins) - [Done](#done) @@ -51,16 +51,6 @@ check it), **Action** (what to do once unblocked), and optional released flopy until then; flopy imports lazily so this is not an import-time problem. -### conda-forge feedstock: pyprms floor replaces packaging pin - -- **Blocked on:** feedstock PR #14 (pyprms >=0.10.0 floor, build 1) - MERGED 2026-08-19; awaiting the main-branch CI upload. Check: - `https://api.anaconda.org/package/conda-forge/pywatershed` shows a - 3.0.0 build with build number 1. -- **Action:** verify the new build's run deps carry `pyprms >=0.10.0` - and no `packaging <26.3`, then move to Done. This was the last - `packaging <26.3` pin anywhere. - ### pyPRMS upstream: close PR #64, port its regression test - **Blocked on:** nothing — #64 was closed unmerged 2026-08-17; the @@ -78,6 +68,32 @@ check it), **Action** (what to do once unblocked), and optional retire `check_version.yaml` in favor of the preflight + release.yaml checks. +### Retire the GSFLOW ifort binary + +- **Blocked on:** nothing external -- this is the follow-up half of the + ifort retirement. Check: `git ls-files bin/` still lists + `gsflow_2.4.0_ifort_apple_silicon_dbl_prec`. +- **Action:** decide what replaces it. GSFLOW source is not in this + repository (`gsflow_src/` is gitignored), so unlike PRMS it cannot be + compiled on demand; it is the last intel-built artifact here, and it is + an **x86_64** binary, so on Apple Silicon it runs under Rosetta 2 -- + including on the `macos-latest` runners for the three + `test_fgr_ag_2yr_*` jobs, which are the only consumers. Options: + vendor/point at a GSFLOW source and build it with gfortran the way PRMS + now is; ship a gfortran-built arm64 GSFLOW binary; or drop the binary + and remove `macos-latest` from the `test_fgr_ag_2yr_*` matrices. +- **Notes:** `bin/gsflow_2.4.0_ifort_apple_silicon_dbl_prec_og` is + byte-identical to the non-`_og` file (verified with `cmp`) and is + referenced nowhere -- 11.9 MB of pure duplicate to delete whichever way + the decision goes. The PRMS half of this work (binaries compiled on + demand from `prms_src/`, ifort and Intel MacOS dropped, `m1` -> `mac_arm`) + landed on branch `feat_drop_ifort`. That compile-on-demand scheme is an + accepted intermediate step: the intended end state is to move the + PRMS/GSFLOW sources into a separate "oracle" repository that publishes + binaries for pywatershed to download, which would settle this item too. + Prefer a cheap stopgap here over a design that assumes the sources stay + in this repository. + ### Deliver CI-usage findings to org admins - **Blocked on:** nothing; conversational. @@ -92,6 +108,11 @@ check it), **Action** (what to do once unblocked), and optional ## Done +- 2026-08-19: conda-forge pywatershed 3.0.0 build 1 published (feedstock + PR #14: `pyprms >=0.10.0` replaces `packaging <26.3`; verified on + anaconda.org, uploaded 19:35 UTC). The last `packaging <26.3` pin + anywhere is gone. Also: PR #407's sagehen ubuntu OOM resolved by + serial pytest (`-n=1`) on Linux. - 2026-08-19: gh-pages v3.0.0 extended release notes are live upstream (`doi-usgs.github.io/pywatershed`, 2026-07-13 post) and all 3.0.0 release-body links resolve — step 9 done, the 3.0.0 release is diff --git a/autotest/ci_local.sh b/autotest/ci_local.sh index e49693bd..17635950 100755 --- a/autotest/ci_local.sh +++ b/autotest/ci_local.sh @@ -98,84 +98,22 @@ echo "" start_dir=$(pwd) -# Function to compile PRMS 5.2.1.1 if binary doesn't exist -compile_prms_5211_if_needed() { - # Determine the binary name based on platform - case "$OSTYPE" in - darwin*) - binary_name="prms_5.2.1.1_gfortran_apple_silicon_dbl_prec" - ;; - linux*) - binary_name="prms_5.2.1.1_gfort_linux_dbl_prec" - ;; - msys* | cygwin* | win32) - binary_name="prms_5.2.1.1_gfort_win_dbl_prec.exe" - ;; - *) - echo "Unknown OS type: $OSTYPE" - return 1 - ;; - esac - - binary_path="../bin/$binary_name" - - # Check if binary exists - if [ -f "$binary_path" ]; then - echo "PRMS 5.2.1.1 binary exists: $binary_path" - return 0 - fi +# Compile a PRMS version from prms_src/ if its binary isn't in ../bin. +# The binary name and the build itself live in pywatershed.utils so that +# this script, ci.yaml, and the test fixtures all agree. +compile_prms_if_needed() { + version="$1" echo "" echo "******************************" - echo "Compiling PRMS 5.2.1.1" + echo "PRMS $version binary" echo "******************************" - echo "Binary not found: $binary_path" - echo "Compiling from source..." - echo "" - - # Save current directory - orig_dir=$(pwd) - - # Navigate to PRMS source directory - cd ../prms_src/prms5.2.1.1 || return 1 - - # Get make path - MAKE_PATH=$(which make) - if [ -z "$MAKE_PATH" ]; then - echo "Error: make not found in PATH" - cd "$orig_dir" - return 1 - fi - - # Compile - echo "Using make: $MAKE_PATH" + echo "Using make: $(which make)" echo "Using gfortran: $(which gfortran)" - gfortran --version - - $MAKE_PATH clean MAKE="$MAKE_PATH" || { - cd "$orig_dir" - return 1 - } - $MAKE_PATH DBL_PREC=true FC=gfortran CC=gcc MAKE="$MAKE_PATH" || { - cd "$orig_dir" - return 1 - } - - # Copy binary to bin directory - if [ -f "bin/prms" ]; then - cp bin/prms "../../bin/$binary_name" || { - cd "$orig_dir" - return 1 - } - echo "Successfully compiled and copied binary to ../../bin/$binary_name" - else - echo "Error: Compilation succeeded but bin/prms not found" - cd "$orig_dir" - return 1 - fi + gfortran --version | head -1 - # Return to original directory - cd "$orig_dir" || return 1 + python -c "import pywatershed as pws; pws.utils.compile_prms('$version')" \ + || return 1 echo "" return 0 @@ -349,6 +287,8 @@ if [ -z "${t}" ]; then echo "DOMAIN: sagehen_5yr" echo "====================" echo + + compile_prms_if_needed 5.2.1 || exit 1 if [ -z "${g}" ]; then echo echo ".........." @@ -416,6 +356,8 @@ if [ -z "${t}" ]; then echo "DOMAIN: hru_1" echo "====================" echo + + compile_prms_if_needed 5.2.1 || exit 1 if [ -z "${g}" ]; then echo @@ -490,8 +432,9 @@ if [ -z "${t}" ]; then echo "====================" echo - # Compile PRMS 5.2.1.1 if binary doesn't exist - compile_prms_5211_if_needed || exit 1 + # Compile the PRMS binaries this domain needs, if absent + compile_prms_if_needed 5.2.1 || exit 1 + compile_prms_if_needed 5.2.1.1 || exit 1 if [ -z "${g}" ]; then echo @@ -607,6 +550,8 @@ if [ -z "${t}" ]; then echo "DOMAIN: ucb_2yr" echo "====================" echo + + compile_prms_if_needed 5.2.1 || exit 1 if [ -z "${g}" ]; then echo echo ".........." diff --git a/autotest/utils.py b/autotest/utils.py index 02f62adc..47a39e54 100644 --- a/autotest/utils.py +++ b/autotest/utils.py @@ -67,23 +67,10 @@ def assert_dicts_equal(dic1, dic2): def detect_prms_exe(): - import sys - from platform import processor - - platform = sys.platform.lower() - if platform == "win32": - exe_name = "prms_win_gfort_dbl_prec.exe" - elif platform == "darwin": - if processor() == "arm": - exe_name = "prms_mac_m1_ifort_dbl_prec" - else: - exe_name = "prms_mac_intel_gfort_dbl_prec" - elif platform == "linux": - exe_name = "prms_linux_gfort_dbl_prec" - else: - exe_name = "---" # this will raise an error - exe_pth = pl.Path(f"../bin/{exe_name}") - return exe_pth + """Path to the default PRMS binary, compiling it from source if absent.""" + from pywatershed.utils import get_or_compile_prms_exe + + return get_or_compile_prms_exe() def run_prms( diff --git a/bin/prms_linux_gfort_dbl_prec b/bin/prms_linux_gfort_dbl_prec deleted file mode 100755 index df8801e2..00000000 Binary files a/bin/prms_linux_gfort_dbl_prec and /dev/null differ diff --git a/bin/prms_linux_gfort_mixed_prec b/bin/prms_linux_gfort_mixed_prec deleted file mode 100755 index d0602a8c..00000000 Binary files a/bin/prms_linux_gfort_mixed_prec and /dev/null differ diff --git a/bin/prms_mac_intel_gfort_dbl_prec b/bin/prms_mac_intel_gfort_dbl_prec deleted file mode 100755 index 23db984e..00000000 Binary files a/bin/prms_mac_intel_gfort_dbl_prec and /dev/null differ diff --git a/bin/prms_mac_intel_gfort_mixed_prec b/bin/prms_mac_intel_gfort_mixed_prec deleted file mode 100755 index d8adcbc5..00000000 Binary files a/bin/prms_mac_intel_gfort_mixed_prec and /dev/null differ diff --git a/bin/prms_mac_m1_ifort_dbl_prec b/bin/prms_mac_m1_ifort_dbl_prec deleted file mode 100755 index 3410b374..00000000 Binary files a/bin/prms_mac_m1_ifort_dbl_prec and /dev/null differ diff --git a/bin/prms_mac_m1_ifort_mixed_prec b/bin/prms_mac_m1_ifort_mixed_prec deleted file mode 100755 index 56403fae..00000000 Binary files a/bin/prms_mac_m1_ifort_mixed_prec and /dev/null differ diff --git a/bin/prms_win_gfort_dbl_prec.exe b/bin/prms_win_gfort_dbl_prec.exe deleted file mode 100644 index f6b139a3..00000000 Binary files a/bin/prms_win_gfort_dbl_prec.exe and /dev/null differ diff --git a/doc/api/utils.rst b/doc/api/utils.rst index 5032df33..df368302 100644 --- a/doc/api/utils.rst +++ b/doc/api/utils.rst @@ -11,6 +11,10 @@ Utils ControlVariables MmrToMf6Dfw utils.cbh_file_to_netcdf + utils.compile_prms utils.DomainSubset + utils.get_or_compile_prms_exe + utils.get_prms_exe_name + utils.get_prms_exe_path utils.netcdf_utils.subset_netcdf_file utils.netcdf_utils.subset_xr diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4b076da2..73909c5f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -27,6 +27,17 @@ Bug fixes Internal changes ~~~~~~~~~~~~~~~~ +- Retire ifort and Intel MacOS. The PRMS binaries are no longer checked in to + ``bin/``; they are compiled from ``prms_src/`` with gfortran (supplied by + ``environment.yml``) the first time they are needed, by + ``pywatershed.utils.compile_prms()``, which ``ci.yaml``, + ``autotest/ci_local.sh``, and the test-data fixtures all call. The + ``prms_src`` makelists gained ``-std=gnu17`` for gcc, without which PRMS + 5.2.1's C sources fail to build under the C23 default of gcc 15. Apple + Silicon binaries are tagged ``mac_arm`` rather than ``m1``, and Intel MacOS + is no longer detected. GSFLOW binaries remain checked in because their + source is not part of this repository. + (:pull:`XXX`) By `James McCreight `_. - Add ``MAINTENANCE.md``, a ledger of maintenance todos blocked on external events (dependency releases, cross-repo work), each with a mechanically checkable unblock condition; the ``/maintenance`` Claude skill checks them diff --git a/evaluation/performance/prms_5.2.1_performance.ipynb b/evaluation/performance/prms_5.2.1_performance.ipynb index d566a33a..c39a7f19 100644 --- a/evaluation/performance/prms_5.2.1_performance.ipynb +++ b/evaluation/performance/prms_5.2.1_performance.ipynb @@ -20,17 +20,26 @@ "import os\n", "import pathlib as pl\n", "import shutil\n", + "\n", "from flopy import run_model\n", "\n", + "import pywatershed as pws\n", "from pywatershed.constants import __pywatershed_root__ as pws_root\n", "\n", "repo_root = pws_root.parent\n", "data_dir = pl.Path(\"../../../data/\")\n", "\n", - "def time_prms_run(domain_dir, exe, working_dir, io, overwrite=False, ):\n", - " \n", + "\n", + "def time_prms_run(\n", + " domain_dir,\n", + " exe,\n", + " working_dir,\n", + " io,\n", + " overwrite=False,\n", + "):\n", + "\n", " domain_dir = pl.Path(domain_dir)\n", - " \n", + "\n", " working_dir = pl.Path(working_dir)\n", " if not working_dir.exists():\n", " working_dir.mkdir(parents=True)\n", @@ -38,26 +47,34 @@ " shutil.rmtree(working_dir)\n", " working_dir.mkdir(parents=True)\n", " else:\n", - " raise FileExistsError(f\"working_dir exists and overwrite=False: {working_dir}\")\n", + " raise FileExistsError(\n", + " f\"working_dir exists and overwrite=False: {working_dir}\"\n", + " )\n", "\n", - " control_dir = data_dir / 'pynhm/performance_runs/PRMS/control_files')\n", - " n_hru_desc = 'multi'\n", - " if domain_dir.name == 'hru_1':\n", - " n_hru_desc = 'single'\n", + " control_dir = data_dir / \"pynhm/performance_runs/PRMS/control_files\"\n", + " n_hru_desc = \"multi\"\n", + " if domain_dir.name == \"hru_1\":\n", + " n_hru_desc = \"single\"\n", " control_file_in = control_dir / f\"control.{n_hru_desc}_hru_{io}_io\"\n", - " control_file = working_dir / 'control.test'\n", - " shutil.copy2(control_file_in, control_file) \n", + " control_file = working_dir / \"control.test\"\n", + " shutil.copy2(control_file_in, control_file)\n", " print(control_file_in)\n", " assert control_file.exists()\n", - " \n", + "\n", " # the control specifies the output here\n", - " (working_dir / 'output').mkdir()\n", - " \n", - " # Copy the requisite files \n", - " req_files = ['myparam.param', 'prcp.cbh', 'tmax.cbh', 'tmin.cbh', 'sf_data']\n", + " (working_dir / \"output\").mkdir()\n", + "\n", + " # Copy the requisite files\n", + " req_files = [\n", + " \"myparam.param\",\n", + " \"prcp.cbh\",\n", + " \"tmax.cbh\",\n", + " \"tmin.cbh\",\n", + " \"sf_data\",\n", + " ]\n", " for ff in req_files:\n", " shutil.copy2(domain_dir / ff, working_dir / ff)\n", - " \n", + "\n", " # print(f\"Running '{control_file}' in {working_dir}\\n\",flush=True,)\n", " def run_model_time():\n", " success, buff = run_model(\n", @@ -74,14 +91,13 @@ " assert success, f\"could not run prms model in '{working_dir}'\"\n", " return\n", "\n", - " \n", - " if \"conus\" in str(domain_dir): \n", + " if \"conus\" in str(domain_dir):\n", " result = %timeit -o -n1 -r1 run_model_nhm()\n", " else:\n", " result = %timeit -o run_model_time()\n", "\n", " return result\n", - " #print(f\"Ran in {working_dir}\\n\", flush=True)" + " # print(f\"Ran in {working_dir}\\n\", flush=True)" ] }, { @@ -94,19 +110,19 @@ "# domain_dir: run_dir/working_dir\n", "\n", "prms_run_dir = data_dir / \"pywatershed/performance_runs/PRMS\"\n", - "test_dom_dir = repo_root / 'test_data/'\n", + "test_dom_dir = repo_root / \"test_data/\"\n", "conus_dom_dir = data_dir / \"pynhm/conus_2yr\"\n", "\n", "dom_dirs = [\n", - " # test_dom_dir / 'hru_1', \n", - " # test_dom_dir / 'drb_2yr', \n", + " # test_dom_dir / 'hru_1',\n", + " # test_dom_dir / 'drb_2yr',\n", " # test_dom_dir / 'ucb_2yr',\n", " conus_dom_dir,\n", "]\n", "\n", "run_dict = {}\n", "for dom in dom_dirs:\n", - " for io in ['no', 'yes']:\n", + " for io in [\"no\", \"yes\"]:\n", " run_dir = f\"{dom.name}_io_{io}\"\n", " src_dir = f\"{dom}\"\n", " run_dict[f\"{prms_run_dir / run_dir}\"] = src_dir" @@ -129,14 +145,20 @@ "metadata": {}, "outputs": [], "source": [ - "exe = repo_root / 'bin/prms_mac'\n", + "exe = pws.utils.get_or_compile_prms_exe()\n", "results = {}\n", "for run_dir, src_dir in run_dict.items():\n", " print(run_dir)\n", - " io = 'yes'\n", - " if run_dir[-2:] == 'no': \n", - " io = 'no'\n", - " results[run_dir] = time_prms_run(domain_dir=src_dir, exe=exe, working_dir=run_dir, io=io, overwrite=True)" + " io = \"yes\"\n", + " if run_dir[-2:] == \"no\":\n", + " io = \"no\"\n", + " results[run_dir] = time_prms_run(\n", + " domain_dir=src_dir,\n", + " exe=exe,\n", + " working_dir=run_dir,\n", + " io=io,\n", + " overwrite=True,\n", + " )" ] }, { @@ -157,11 +179,14 @@ "outputs": [], "source": [ "import pickle\n", + "\n", "for path, result in results.items():\n", " path = pl.Path(path)\n", - " pkl_path = path.parent.parent / (f\"results/{path.parent.name}_{path.name}.pkl\")\n", + " pkl_path = path.parent.parent / (\n", + " f\"results/{path.parent.name}_{path.name}.pkl\"\n", + " )\n", " print(pkl_path)\n", - " \n", + "\n", " with open(pkl_path, \"wb\") as output_file:\n", " pickle.dump(result, output_file)" ] @@ -173,7 +198,7 @@ "metadata": {}, "outputs": [], "source": [ - "import os; os.getcwd()" + "os.getcwd()" ] }, { @@ -184,8 +209,8 @@ "outputs": [], "source": [ "results2 = {}\n", - "files = pl.Path('../../../data/pynhm/performance_runs/results/').glob('*.pkl')\n", - "for ff in files: \n", + "files = pl.Path(\"../../../data/pynhm/performance_runs/results/\").glob(\"*.pkl\")\n", + "for ff in files:\n", " print(ff)\n", " with open(ff, \"rb\") as input_file:\n", " results2[ff.name[0:-4]] = pickle.load(input_file)" @@ -208,7 +233,7 @@ "metadata": {}, "outputs": [], "source": [ - "results2['PRMS_drb_2yr_io_yes'].average\n" + "results2[\"PRMS_drb_2yr_io_yes\"].average" ] }, { @@ -219,6 +244,8 @@ "outputs": [], "source": [ "import time\n", + "\n", + "\n", "def test():\n", " time.sleep(4)\n", " return 10" diff --git a/examples/runoff_errors.ipynb b/examples/runoff_errors.ipynb index 71863130..a0668143 100644 --- a/examples/runoff_errors.ipynb +++ b/examples/runoff_errors.ipynb @@ -29,15 +29,15 @@ "outputs": [], "source": [ "import pathlib as pl\n", - "from pprint import pprint\n", - "from shutil import rmtree, copy2\n", + "from shutil import copy2\n", "\n", "import hvplot.xarray # noqa\n", - "from IPython.display import display\n", "import jupyter_black\n", "import numpy as np\n", - "import pywatershed as pws\n", "import xarray as xr\n", + "from IPython.display import display\n", + "\n", + "import pywatershed as pws\n", "\n", "jupyter_black.load()" ] @@ -86,9 +86,9 @@ "metadata": {}, "outputs": [], "source": [ - "bin_dir = pws_root / \"../bin/\"\n", - "# bin_mixed = bin_dir / \"prms_mac_m1_ifort_mixed_prec\"\n", - "bin_double = bin_dir / \"prms_mac_m1_ifort_dbl_prec\"" + "# The PRMS binary is compiled from prms_src/ the first time it\n", + "# is needed; nothing is checked in to bin/.\n", + "bin_double = pws.utils.get_or_compile_prms_exe()" ] }, { @@ -99,15 +99,14 @@ "outputs": [], "source": [ "def run_prms(binary: pl.Path, run_dir: pl.Path, skip_if_exists=False):\n", - " import shlex\n", " import subprocess\n", "\n", - " from pywatershed import CsvFile, Soltab\n", - " from pywatershed.parameters import PrmsParameters\n", + " from pywatershed import CsvFile\n", "\n", " if skip_if_exists and run_dir.exists():\n", " print(\n", - " f\"Run ({run_dir}) already exists and skip_if_exists=True. Using existing run.\"\n", + " f\"Run ({run_dir}) already exists and \"\n", + " \"skip_if_exists=True. Using existing run.\"\n", " )\n", " return None\n", "\n", @@ -126,8 +125,11 @@ " output_dir = run_dir / \"output\"\n", " output_dir.mkdir()\n", "\n", - " exe_command = f\"time ./{binary.name} nhm.control -MAXDATALNLEN 60000 2>&1 | tee run.log\"\n", - " result = subprocess.run(\n", + " exe_command = (\n", + " f\"time ./{binary.name} nhm.control \"\n", + " \"-MAXDATALNLEN 60000 2>&1 | tee run.log\"\n", + " )\n", + " subprocess.run(\n", " exe_command,\n", " shell=True,\n", " # stdout = subprocess.PIPE,\n", @@ -334,7 +336,8 @@ "source": [ "if output_dir.exists() and skip_if_exists_pws:\n", " print(\n", - " f\"Output ({output_dir}) already exists and skip_if_exists=True. Using existing run.\"\n", + " f\"Output ({output_dir}) already exists and \"\n", + " \"skip_if_exists=True. Using existing run.\"\n", " )\n", "\n", "else:\n", @@ -373,7 +376,7 @@ " assert (output_dir / f\"{vv}.nc\").exists()\n", " try:\n", " assert (input_dir_cp / f\"{vv}.nc\").exists()\n", - " except:\n", + " except AssertionError:\n", " print(f\"********** {vv} not in input_dir_cp\")" ] }, @@ -668,13 +671,13 @@ "\n", "print(\"-----------\")\n", "\n", - "print(f'{bc[\"through_rain\"].pws.values=}')\n", + "print(f\"{bc[\"through_rain\"].pws.values=}\")\n", "\n", - "print(f'{bc[\"snow_evap\"].prms.values=}')\n", - "print(f'{bc[\"hru_impervstor_change\"].prms.values=}')\n", - "print(f'{bc[\"hru_impervstor_change\"].pws.values=}')\n", - "print(f'{bc[\"dprst_stor_hru_change\"].prms.values=}')\n", - "print(f'{bc[\"dprst_stor_hru_change\"].pws.values=}')\n", + "print(f\"{bc[\"snow_evap\"].prms.values=}\")\n", + "print(f\"{bc[\"hru_impervstor_change\"].prms.values=}\")\n", + "print(f\"{bc[\"hru_impervstor_change\"].pws.values=}\")\n", + "print(f\"{bc[\"dprst_stor_hru_change\"].prms.values=}\")\n", + "print(f\"{bc[\"dprst_stor_hru_change\"].pws.values=}\")\n", "print(f\"{balance.prms.values=}\")\n", "\n", "# print(f\"{bc[\"hru_sroffi\"].prms.sum().values=}\")\n", @@ -695,8 +698,8 @@ "metadata": {}, "outputs": [], "source": [ - "print(f'{(balance - bc[\"through_rain\"]).pws.values=}')\n", - "print(f'{(balance - bc[\"through_rain\"]).prms.values=}')" + "print(f\"{(balance - bc[\"through_rain\"]).pws.values=}\")\n", + "print(f\"{(balance - bc[\"through_rain\"]).prms.values=}\")" ] }, { @@ -726,14 +729,15 @@ "metadata": {}, "outputs": [], "source": [ - "print(f'{bc[\"through_rain\"].pws.values=}')\n", - "print(f'{bc[\"net_rain\"].pws.values=}')\n", - "print(f'{bc[\"net_snow\"].pws.values=}')\n", - "print(f'{bc[\"net_ppt\"].pws.values=}')\n", - "print(f'{bc[\"pptmix_nopack\"].pws.values=}')\n", - "print(f'{bc[\"newsnow\"].pws.values=}')\n", + "print(f\"{bc[\"through_rain\"].pws.values=}\")\n", + "print(f\"{bc[\"net_rain\"].pws.values=}\")\n", + "print(f\"{bc[\"net_snow\"].pws.values=}\")\n", + "print(f\"{bc[\"net_ppt\"].pws.values=}\")\n", + "print(f\"{bc[\"pptmix_nopack\"].pws.values=}\")\n", + "print(f\"{bc[\"newsnow\"].pws.values=}\")\n", "print(\n", - " f'{(bc[\"pk_ice_prev\"].pws.values + bc[\"freeh2o_prev\"].pws.values) < epsilon32=}'\n", + " f\"{(bc[\"pk_ice_prev\"].pws.values\n", + " + bc[\"freeh2o_prev\"].pws.values) < epsilon32=}\"\n", ")" ] }, diff --git a/examples/snow_errors.ipynb b/examples/snow_errors.ipynb index 642309dc..e94bc92f 100644 --- a/examples/snow_errors.ipynb +++ b/examples/snow_errors.ipynb @@ -29,15 +29,15 @@ "outputs": [], "source": [ "import pathlib as pl\n", - "from pprint import pprint\n", - "from shutil import rmtree, copy2\n", + "from shutil import copy2\n", "\n", "import hvplot.xarray # noqa\n", "import jupyter_black\n", - "from IPython.display import display\n", "import numpy as np\n", - "import pywatershed as pws\n", "import xarray as xr\n", + "from IPython.display import display\n", + "\n", + "import pywatershed as pws\n", "\n", "jupyter_black.load()" ] @@ -89,9 +89,9 @@ "metadata": {}, "outputs": [], "source": [ - "bin_dir = pws_root / \"../bin/\"\n", - "# bin_mixed = bin_dir / \"prms_521_mixed_mac_m1_intel\"\n", - "bin_double = bin_dir / \"prms_mac_m1_ifort_dbl_prec\"" + "# The PRMS binary is compiled from prms_src/ the first time it\n", + "# is needed; nothing is checked in to bin/.\n", + "bin_double = pws.utils.get_or_compile_prms_exe()" ] }, { @@ -102,16 +102,15 @@ "outputs": [], "source": [ "def run_prms(binary: pl.Path, run_dir: pl.Path, skip_if_exists=False):\n", - " import shlex\n", " import subprocess\n", "\n", " from pywatershed import CsvFile, Soltab\n", - "\n", " from pywatershed.parameters import PrmsParameters\n", "\n", " if skip_if_exists and run_dir.exists():\n", " print(\n", - " f\"Run ({run_dir}) already exists and skip_if_exists=True. Using existing run.\"\n", + " f\"Run ({run_dir}) already exists and \"\n", + " \"skip_if_exists=True. Using existing run.\"\n", " )\n", " return None\n", "\n", @@ -130,8 +129,11 @@ " output_dir = run_dir / \"output\"\n", " output_dir.mkdir()\n", "\n", - " exe_command = f\"time ./{binary.name} nhm.control -MAXDATALNLEN 60000 2>&1 | tee run.log\"\n", - " result = subprocess.run(\n", + " exe_command = (\n", + " f\"time ./{binary.name} nhm.control \"\n", + " \"-MAXDATALNLEN 60000 2>&1 | tee run.log\"\n", + " )\n", + " subprocess.run(\n", " exe_command,\n", " shell=True,\n", " # stdout = subprocess.PIPE,\n", @@ -316,7 +318,8 @@ "source": [ "if output_dir.exists() and skip_if_exists_pws:\n", " print(\n", - " f\"Output ({output_dir}) already exists and skip_if_exists=True. Using existing run.\"\n", + " f\"Output ({output_dir}) already exists and \"\n", + " \"skip_if_exists=True. Using existing run.\"\n", " )\n", "\n", "else:\n", @@ -347,7 +350,7 @@ " assert (output_dir / f\"{vv}.nc\").exists()\n", " try:\n", " assert (input_dir / f\"{vv}.nc\").exists()\n", - " except:\n", + " except AssertionError:\n", " print(f\"********** {vv} not in input_dir\")" ] }, diff --git a/prms_src/prms5.2.1.1/makelist b/prms_src/prms5.2.1.1/makelist index 634767a7..1e31ec47 100644 --- a/prms_src/prms5.2.1.1/makelist +++ b/prms_src/prms5.2.1.1/makelist @@ -90,14 +90,19 @@ endif # Define the C compile flags # -D_UF defines UNIX naming conventions for mixed language compilation. ########################################################## +# -std=gnu17: the C sources use K&R-style prototypes such as +# `static char *open_parameter_file();` and then call them with +# arguments. Under C23 -- the default since gcc 15 / clang 15 -- an +# empty parameter list means "takes no arguments", so those calls +# become hard errors. gnu17 keeps the old meaning. # set the C flags ifeq ($(detected_OS), Windows) ifeq ($(CC), gcc) - CFLAGS = $(OPTLEVEL) -D$(ARC) -D_UF -Wall + CFLAGS = $(OPTLEVEL) -D$(ARC) -D_UF -Wall -std=gnu17 endif else ifeq ($(CC), gcc) - CFLAGS = $(OPTLEVEL) -D$(ARC) -D_UF -Wall + CFLAGS = $(OPTLEVEL) -D$(ARC) -D_UF -Wall -std=gnu17 endif ifeq ($(CC), $(filter $(CC), icc mpiicc)) CFLAGS = $(OPTLEVEL) -D$(ARC) -D_UF -Wall diff --git a/prms_src/prms5.2.1.1/makelist_double_precision b/prms_src/prms5.2.1.1/makelist_double_precision index 223f23cc..fd985a52 100644 --- a/prms_src/prms5.2.1.1/makelist_double_precision +++ b/prms_src/prms5.2.1.1/makelist_double_precision @@ -93,17 +93,22 @@ endif # Define the C compile flags # -D_UF defines UNIX naming conventions for mixed language compilation. ########################################################## +# -std=gnu17: the C sources use K&R-style prototypes such as +# `static char *open_parameter_file();` and then call them with +# arguments. Under C23 -- the default since gcc 15 / clang 15 -- an +# empty parameter list means "takes no arguments", so those calls +# become hard errors. gnu17 keeps the old meaning. # set the C flags # CFLTDBL is C floats to double: promote all floats to 64bit # This is a shortcut to alleviate errors when parsing parameter inputs in soltab. ifeq ($(detected_OS), Windows) ifeq ($(CC), gcc) - CFLAGS = -Dfloat=double $(OPTLEVEL) -D$(ARC) -D_UF -Wall + CFLAGS = -Dfloat=double $(OPTLEVEL) -D$(ARC) -D_UF -Wall -std=gnu17 endif else ifeq ($(CC), gcc) - CFLAGS = -Dfloat=double $(OPTLEVEL) -D$(ARC) -D_UF -Wall + CFLAGS = -Dfloat=double $(OPTLEVEL) -D$(ARC) -D_UF -Wall -std=gnu17 endif ifeq ($(CC), $(filter $(CC), icc mpiicc)) CFLAGS = $(OPTLEVEL) -Dfloat=double -D$(ARC) -D_UF -Wall diff --git a/prms_src/prms5.2.1/makelist b/prms_src/prms5.2.1/makelist index b4b0e1e6..132ae4c6 100644 --- a/prms_src/prms5.2.1/makelist +++ b/prms_src/prms5.2.1/makelist @@ -91,14 +91,19 @@ endif # Define the C compile flags # -D_UF defines UNIX naming conventions for mixed language compilation. ########################################################## +# -std=gnu17: the C sources use K&R-style prototypes such as +# `static char *open_parameter_file();` and then call them with +# arguments. Under C23 -- the default since gcc 15 / clang 15 -- an +# empty parameter list means "takes no arguments", so those calls +# become hard errors. gnu17 keeps the old meaning. # set the C flags ifeq ($(detected_OS), Windows) ifeq ($(CC), gcc) - CFLAGS = $(OPTLEVEL) -D$(ARC) -D_UF -Wall + CFLAGS = $(OPTLEVEL) -D$(ARC) -D_UF -Wall -std=gnu17 endif else ifeq ($(CC), gcc) - CFLAGS = $(OPTLEVEL) -D$(ARC) -D_UF -Wall + CFLAGS = $(OPTLEVEL) -D$(ARC) -D_UF -Wall -std=gnu17 endif ifeq ($(CC), $(filter $(CC), icc mpiicc)) CFLAGS = $(OPTLEVEL) -D$(ARC) -D_UF -Wall diff --git a/prms_src/prms5.2.1/makelist_double_precision b/prms_src/prms5.2.1/makelist_double_precision index 223f23cc..fd985a52 100644 --- a/prms_src/prms5.2.1/makelist_double_precision +++ b/prms_src/prms5.2.1/makelist_double_precision @@ -93,17 +93,22 @@ endif # Define the C compile flags # -D_UF defines UNIX naming conventions for mixed language compilation. ########################################################## +# -std=gnu17: the C sources use K&R-style prototypes such as +# `static char *open_parameter_file();` and then call them with +# arguments. Under C23 -- the default since gcc 15 / clang 15 -- an +# empty parameter list means "takes no arguments", so those calls +# become hard errors. gnu17 keeps the old meaning. # set the C flags # CFLTDBL is C floats to double: promote all floats to 64bit # This is a shortcut to alleviate errors when parsing parameter inputs in soltab. ifeq ($(detected_OS), Windows) ifeq ($(CC), gcc) - CFLAGS = -Dfloat=double $(OPTLEVEL) -D$(ARC) -D_UF -Wall + CFLAGS = -Dfloat=double $(OPTLEVEL) -D$(ARC) -D_UF -Wall -std=gnu17 endif else ifeq ($(CC), gcc) - CFLAGS = -Dfloat=double $(OPTLEVEL) -D$(ARC) -D_UF -Wall + CFLAGS = -Dfloat=double $(OPTLEVEL) -D$(ARC) -D_UF -Wall -std=gnu17 endif ifeq ($(CC), $(filter $(CC), icc mpiicc)) CFLAGS = $(OPTLEVEL) -Dfloat=double -D$(ARC) -D_UF -Wall diff --git a/pywatershed/utils/prms_exe_utils.py b/pywatershed/utils/prms_exe_utils.py index b110ef4a..ba70ba66 100644 --- a/pywatershed/utils/prms_exe_utils.py +++ b/pywatershed/utils/prms_exe_utils.py @@ -2,15 +2,35 @@ The binary naming conventions mirror those in test_data/generate/conftest.py. Compilation logic mirrors autotest/ci_local.sh. + +PRMS binaries are not kept in the repository; they are compiled from the +sources in ``prms_src/`` with gfortran (supplied by environment.yml) the +first time they are needed. GSFLOW is the exception: its source is not +part of this repository, so its binaries are checked in. + +Supported platforms are Windows, Linux, and Apple Silicon macOS. Intel +macOS is no longer supported. """ +import contextlib +import os import pathlib as pl import shutil import subprocess import sys -from platform import processor from typing import Optional +#: PRMS versions in ``prms_src/`` that can be compiled from source, mapped +#: to their source directory names. +COMPILABLE_SOURCES = { + "5.2.1": "prms5.2.1", + "5.2.1.1": "prms5.2.1.1", +} + +#: The version behind the plain "prms" executable, i.e. control files whose +#: ``executable_desc`` is "PRMS 4" or absent. +DEFAULT_PRMS_SOURCE = "5.2.1" + def _get_repo_root() -> pl.Path: from pywatershed.utils.notebook_utils import get_repo_root @@ -26,6 +46,52 @@ def _get_src_dir() -> pl.Path: return _get_repo_root() / "prms_src" +def _get_platform_tag() -> str: + """Return the platform tag used in binary file names. + + Returns + ------- + str + One of ``"win"``, ``"mac_arm"``, or ``"linux"``. + + Raises + ------ + ValueError + When the platform is not recognised. + """ + platform = sys.platform.lower() + if platform == "win32": + return "win" + elif platform == "darwin": + return "mac_arm" + elif platform == "linux": + return "linux" + else: + raise ValueError(f"Unsupported platform: {platform}") + + +def _get_source_for_exe_desc(exe_desc: str) -> Optional[str]: + """Return the compilable source version for *exe_desc*, or None. + + Parameters + ---------- + exe_desc : str + See :func:`get_prms_exe_name`. + + Returns + ------- + str or None + A key of :data:`COMPILABLE_SOURCES`, or ``None`` when *exe_desc* + names an executable this repository cannot build (GSFLOW). + """ + exe_desc_lower = exe_desc.lower() + if "gsflow" in exe_desc_lower: + return None + if "5.2.1.1" in exe_desc_lower: + return "5.2.1.1" + return DEFAULT_PRMS_SOURCE + + def get_prms_exe_name(exe_desc: str = "prms") -> str: """Return the binary filename for the given exe_desc and current platform. @@ -43,59 +109,28 @@ def get_prms_exe_name(exe_desc: str = "prms") -> str: Raises ------ - NotImplementedError - When a pre-built binary is not available for the current platform. ValueError When the platform is not recognised. """ exe_desc_lower = exe_desc.lower() - platform = sys.platform.lower() - proc = processor() + tag = _get_platform_tag() + suffix = ".exe" if tag == "win" else "" if "gsflow" in exe_desc_lower: - if platform == "win32": - return "gsflow_2.4.0_gfortran_windows_dbl_prec.exe" - elif platform == "darwin": - if proc == "arm": - return "gsflow_2.4.0_ifort_apple_silicon_dbl_prec" - else: - raise NotImplementedError( - f"GSFLOW binary not yet provided for {platform}:intel" - ) - elif platform == "linux": - return "gsflow_2.4.0_gfortran_linux_dbl_prec" - else: - raise ValueError(f"Unsupported platform: {platform}") + # GSFLOW source is not in this repository, so these binaries are + # checked in rather than compiled on demand. + return { + "win": "gsflow_2.4.0_gfortran_windows_dbl_prec.exe", + "mac_arm": "gsflow_2.4.0_ifort_apple_silicon_dbl_prec", + "linux": "gsflow_2.4.0_gfortran_linux_dbl_prec", + }[tag] elif "5.2.1.1" in exe_desc_lower: - if platform == "win32": - return "prms_5.2.1.1_gfort_win_dbl_prec.exe" - elif platform == "darwin": - if proc == "arm": - return "prms_5.2.1.1_gfortran_apple_silicon_dbl_prec" - else: - raise NotImplementedError( - "PRMS 5.2.1.1 binary not yet provided for " - f"{platform}:intel" - ) - elif platform == "linux": - return "prms_5.2.1.1_gfort_linux_dbl_prec" - else: - raise ValueError(f"Unsupported platform: {platform}") + return f"prms_5.2.1.1_gfort_{tag}_dbl_prec{suffix}" else: - # Default PRMS binary - if platform == "win32": - return "prms_win_gfort_dbl_prec.exe" - elif platform == "darwin": - if proc == "arm": - return "prms_mac_m1_ifort_dbl_prec" - else: - return "prms_mac_intel_gfort_dbl_prec" - elif platform == "linux": - return "prms_linux_gfort_dbl_prec" - else: - raise ValueError(f"Unsupported platform: {platform}") + # Default PRMS binary, version 5.2.1 + return f"prms_{tag}_gfort_dbl_prec{suffix}" def get_prms_exe_path(exe_desc: str = "prms") -> pl.Path: @@ -114,8 +149,30 @@ def get_prms_exe_path(exe_desc: str = "prms") -> pl.Path: return (_get_bin_dir() / get_prms_exe_name(exe_desc)).resolve() +@contextlib.contextmanager +def _compile_lock(source: str): + """Serialize compilation of *source* across processes. + + ``generate_test_data.py`` runs pytest with ``-n=auto``, so several + workers can ask for the same binary at once. Without a lock they would + run ``make clean`` in the same source tree while another was compiling. + ``filelock`` is a test dependency, not a runtime one; when it is absent + compilation proceeds unserialized, which is correct for a single process. + """ + try: + from filelock import FileLock + except ImportError: + yield + return + + bin_dir = _get_bin_dir() + bin_dir.mkdir(parents=True, exist_ok=True) + with FileLock(str(bin_dir / f".compile_prms_{source}.lock"), timeout=1800): + yield + + def compile_prms( - source: str = "5.2.1.1", + source: str = DEFAULT_PRMS_SOURCE, force: bool = False, ) -> pl.Path: """Compile PRMS from source and return the path to the resulting binary. @@ -123,8 +180,7 @@ def compile_prms( Parameters ---------- source : str - Source version to compile. Currently only ``"5.2.1.1"`` is - supported. + Source version to compile; a key of :data:`COMPILABLE_SOURCES`. force : bool If ``True``, recompile even when the binary already exists. Default is ``False`` (skip if binary present). @@ -143,64 +199,68 @@ def compile_prms( subprocess.CalledProcessError If any subprocess step fails. """ - if source != "5.2.1.1": + if source not in COMPILABLE_SOURCES: raise ValueError( - f"Unsupported source '{source}'. Currently only '5.2.1.1' " - "can be compiled." + f"Unsupported source '{source}'. Supported sources are: " + f"{sorted(COMPILABLE_SOURCES)}." ) binary_path = get_prms_exe_path(source) - src_dir = _get_src_dir() / "prms5.2.1.1" + src_dir = _get_src_dir() / COMPILABLE_SOURCES[source] if binary_path.exists() and not force: print(f"PRMS {source} binary already exists: {binary_path}") return binary_path - print( - f"\n{'*' * 30}\n" - f"Compiling PRMS {source}\n" - f"{'*' * 30}\n" - f"Source dir : {src_dir}\n" - f"Binary dest: {binary_path}\n" - ) - - orig_dir = pl.Path.cwd() - try: - import os - - os.chdir(src_dir) - - # Clean previous build - subprocess.run( - ["make", "clean", "MAKE=make"], - check=True, + with _compile_lock(source): + # Another process may have compiled it while this one waited. + if binary_path.exists() and not force: + print(f"PRMS {source} binary already exists: {binary_path}") + return binary_path + + print( + f"\n{'*' * 30}\n" + f"Compiling PRMS {source}\n" + f"{'*' * 30}\n" + f"Source dir : {src_dir}\n" + f"Binary dest: {binary_path}\n" ) - # Build with double precision - subprocess.run( - [ - "make", - "DBL_PREC=true", - "FC=gfortran", - "CC=gcc", - "MAKE=make", - ], - check=True, - ) + orig_dir = pl.Path.cwd() + try: + os.chdir(src_dir) - compiled_bin = src_dir / "bin" / "prms" - if not compiled_bin.exists(): - raise RuntimeError( - "Compilation appeared to succeed but " - f"bin/prms not found in {src_dir}" + # Clean previous build + subprocess.run( + ["make", "clean", "MAKE=make"], + check=True, ) - _get_bin_dir().mkdir(parents=True, exist_ok=True) - shutil.copy(compiled_bin, binary_path) - print(f"Successfully compiled and installed to {binary_path}") + # Build with double precision + subprocess.run( + [ + "make", + "DBL_PREC=true", + "FC=gfortran", + "CC=gcc", + "MAKE=make", + ], + check=True, + ) - finally: - os.chdir(orig_dir) + compiled_bin = src_dir / "bin" / "prms" + if not compiled_bin.exists(): + raise RuntimeError( + "Compilation appeared to succeed but " + f"bin/prms not found in {src_dir}" + ) + + _get_bin_dir().mkdir(parents=True, exist_ok=True) + shutil.copy(compiled_bin, binary_path) + print(f"Successfully compiled and installed to {binary_path}") + + finally: + os.chdir(orig_dir) return binary_path @@ -212,10 +272,9 @@ def get_or_compile_prms_exe( ) -> pl.Path: """Return the PRMS/GSFLOW executable, compiling from source if needed. - For ``exe_desc`` values that map to a compilable source (currently only - ``"5.2.1.1"``), the binary will be compiled automatically when it is not - present (or when *force* is ``True``). For other variants the binary - must already exist. + PRMS binaries are compiled automatically when they are not present (or + when *force* is ``True``). GSFLOW has no source in this repository, so + its binary must already exist in ``bin/``. Parameters ---------- @@ -246,16 +305,14 @@ def get_or_compile_prms_exe( # Determine which source to compile source = compile_source if source is None: - if "5.2.1.1" in exe_desc.lower(): - source = "5.2.1.1" + source = _get_source_for_exe_desc(exe_desc) if source is not None: return compile_prms(source=source, force=force) - if not exe_path.exists(): - raise FileNotFoundError( - f"Executable not found and no compilable source is available " - f"for exe_desc='{exe_desc}': {exe_path}" - ) - - return exe_path + raise FileNotFoundError( + f"Executable not found and no compilable source is available for " + f"exe_desc='{exe_desc}': {exe_path}\n" + "GSFLOW source is not part of this repository; its binaries are " + "checked in to bin/." + ) diff --git a/test_data/README.md b/test_data/README.md index c80bc088..dc878727 100644 --- a/test_data/README.md +++ b/test_data/README.md @@ -77,8 +77,10 @@ The first and last remove data and are generally only used to support testing in CI. The "run" and "convert" are the core scripts for generating the test data. -The `run_prms_domains.py` scripts uses binaries provided in the pywatershed -repository to run reference models, specifically the PRMS v5.2.1 binary. All +The `run_prms_domains.py` scripts runs reference models, specifically the +PRMS v5.2.1 binary. That binary is not kept in the repository; it is +compiled from `prms_src/prms5.2.1` with gfortran the first time it is +needed and written to `bin/`. All domains included in the the 'test_data/' directory will be run with the binary (unless the `--domain` option is exercised) and the output generated by the binary then becomes test answers or reference data for @@ -86,12 +88,15 @@ pytest autotests. The `convert_prms_output_to_nc.py` simply converts the output from the first step to NetCDF format and also calculates a few derived variables, providing the actual data sources used by `autotest`. -It is possible that the included binaries will not work on your system. -For a failing PRMS 5.2.1 binary, please see [`prms_src/prms5.2.1/README.md`](../prms_src/prms5.2.1/README.md). -If you compile yourself, you'll need to copy `prms_src/prms5.2.1/bin/prms` -to `bin/` while renaming it to one of the existing files found therein. -Currently, supplied binaries are all compiled with gfortran except on -Mac ARM architecture where ifort is used. +The PRMS binaries are compiled on demand by +`pywatershed.utils.compile_prms()`, which builds with gfortran and installs +into `bin/` under the name `pywatershed.utils.get_prms_exe_name()` expects. +To force a rebuild, call it with `force=True`. If a build fails, please see +[`prms_src/prms5.2.1/README.md`](../prms_src/prms5.2.1/README.md). + +The GSFLOW binaries in `bin/` are the exception: GSFLOW source is not part +of this repository, so they are checked in. The Apple Silicon one is an +x86_64 intel build and needs Rosetta 2. ## Domain directories diff --git a/test_data/generate/conftest.py b/test_data/generate/conftest.py index b7d11907..5c5ba9f5 100644 --- a/test_data/generate/conftest.py +++ b/test_data/generate/conftest.py @@ -128,10 +128,10 @@ def exe(simulation, request): return exe_pth exe_desc = get_ctl_exe_desc(simulation["control_file"]) - try: - return pws.utils.get_prms_exe_path(exe_desc) - except NotImplementedError as e: - pytest.skip(str(e)) + # PRMS binaries are compiled from prms_src/ on first use; the GSFLOW + # binaries are checked in to bin/ because their source is not in this + # repository. + return pws.utils.get_or_compile_prms_exe(exe_desc) def scheduler_active():