From ad9bd39ef01b6199dcbfbf9e2c01a1f94c668b9f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Jul 2025 09:03:10 -0600 Subject: [PATCH 001/163] Add spack builtin repo as repos/builtin and migrate spack-ext/repos/spack-stack to new api version 2 under repos/spack_stack --- .gitmodules | 10 ++++++++-- repos/builtin | 1 + .../spack_repo/spack_stack/packages/ai_env}/package.py | 1 + .../spack_stack/packages/base_env}/package.py | 1 + .../spack_stack/packages/emc_gfs_wafs_env}/package.py | 1 + .../packages/emc_verif_global_env}/package.py | 1 + .../spack_repo/spack_stack}/packages/ewok/package.py | 1 + .../spack_stack/packages/ewok_env}/package.py | 1 + .../spack_stack/packages/geos_gcm_env}/package.py | 1 + .../spack_stack/packages/gldas_env}/package.py | 1 + .../packages/global_workflow_env}/package.py | 1 + .../spack_stack/packages/gmao_swell_env}/package.py | 1 + .../spack_stack/packages/gsi_env}/package.py | 1 + .../spack_stack/packages/jedi_base_env}/package.py | 1 + .../spack_stack/packages/jedi_fv3_env}/package.py | 1 + .../spack_stack/packages/jedi_geos_env}/package.py | 1 + .../spack_stack/packages/jedi_mpas_env}/package.py | 1 + .../spack_stack/packages/jedi_neptune_env}/package.py | 1 + .../spack_stack/packages/jedi_tools_env}/package.py | 1 + .../spack_stack/packages/jedi_ufs_env}/package.py | 1 + .../spack_stack/packages/jedi_um_env}/package.py | 1 + .../spack_stack/packages/nceplibs_env}/package.py | 1 + .../spack_stack/packages/neptune_env}/package.py | 1 + .../packages/neptune_python_env}/package.py | 1 + .../spack_stack/packages/py_xnrl}/package.py | 1 + .../spack_repo/spack_stack}/packages/r2d2/package.py | 1 + .../spack_stack}/packages/shumlib/package.py | 1 + .../spack_stack/packages/soca_env}/package.py | 1 + .../spack_repo/spack_stack}/packages/solo/package.py | 1 + .../spack_stack/packages/ufs_pyenv}/package.py | 1 + .../spack_stack/packages/ufs_srw_app_env}/package.py | 1 + .../spack_stack/packages/ufs_utils_env}/package.py | 1 + .../packages/ufs_weather_model_env}/package.py | 1 + .../spack_stack/packages/upp_env}/package.py | 1 + .../spack_stack/packages/ww3_env}/package.py | 1 + repos/spack_stack/spack_repo/spack_stack/repo.yaml | 3 +++ spack-ext/repos/spack-stack/repo.yaml | 2 -- 37 files changed, 45 insertions(+), 4 deletions(-) create mode 160000 repos/builtin rename {spack-ext/repos/spack-stack/packages/ai-env => repos/spack_stack/spack_repo/spack_stack/packages/ai_env}/package.py (91%) rename {spack-ext/repos/spack-stack/packages/base-env => repos/spack_stack/spack_repo/spack_stack/packages/base_env}/package.py (95%) rename {spack-ext/repos/spack-stack/packages/emc-gfs-wafs-env => repos/spack_stack/spack_repo/spack_stack/packages/emc_gfs_wafs_env}/package.py (92%) rename {spack-ext/repos/spack-stack/packages/emc-verif-global-env => repos/spack_stack/spack_repo/spack_stack/packages/emc_verif_global_env}/package.py (94%) rename {spack-ext/repos/spack-stack => repos/spack_stack/spack_repo/spack_stack}/packages/ewok/package.py (94%) rename {spack-ext/repos/spack-stack/packages/ewok-env => repos/spack_stack/spack_repo/spack_stack/packages/ewok_env}/package.py (97%) rename {spack-ext/repos/spack-stack/packages/geos-gcm-env => repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env}/package.py (94%) rename {spack-ext/repos/spack-stack/packages/gldas-env => repos/spack_stack/spack_repo/spack_stack/packages/gldas_env}/package.py (91%) rename {spack-ext/repos/spack-stack/packages/global-workflow-env => repos/spack_stack/spack_repo/spack_stack/packages/global_workflow_env}/package.py (95%) rename {spack-ext/repos/spack-stack/packages/gmao-swell-env => repos/spack_stack/spack_repo/spack_stack/packages/gmao_swell_env}/package.py (97%) rename {spack-ext/repos/spack-stack/packages/gsi-env => repos/spack_stack/spack_repo/spack_stack/packages/gsi_env}/package.py (93%) rename {spack-ext/repos/spack-stack/packages/jedi-base-env => repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env}/package.py (97%) rename {spack-ext/repos/spack-stack/packages/jedi-fv3-env => repos/spack_stack/spack_repo/spack_stack/packages/jedi_fv3_env}/package.py (90%) rename {spack-ext/repos/spack-stack/packages/jedi-geos-env => repos/spack_stack/spack_repo/spack_stack/packages/jedi_geos_env}/package.py (93%) rename {spack-ext/repos/spack-stack/packages/jedi-mpas-env => repos/spack_stack/spack_repo/spack_stack/packages/jedi_mpas_env}/package.py (92%) rename {spack-ext/repos/spack-stack/packages/jedi-neptune-env => repos/spack_stack/spack_repo/spack_stack/packages/jedi_neptune_env}/package.py (91%) rename {spack-ext/repos/spack-stack/packages/jedi-tools-env => repos/spack_stack/spack_repo/spack_stack/packages/jedi_tools_env}/package.py (95%) rename {spack-ext/repos/spack-stack/packages/jedi-ufs-env => repos/spack_stack/spack_repo/spack_stack/packages/jedi_ufs_env}/package.py (93%) rename {spack-ext/repos/spack-stack/packages/jedi-um-env => repos/spack_stack/spack_repo/spack_stack/packages/jedi_um_env}/package.py (92%) rename {spack-ext/repos/spack-stack/packages/nceplibs-env => repos/spack_stack/spack_repo/spack_stack/packages/nceplibs_env}/package.py (94%) rename {spack-ext/repos/spack-stack/packages/neptune-env => repos/spack_stack/spack_repo/spack_stack/packages/neptune_env}/package.py (96%) rename {spack-ext/repos/spack-stack/packages/neptune-python-env => repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env}/package.py (95%) rename {spack-ext/repos/spack-stack/packages/py-xnrl => repos/spack_stack/spack_repo/spack_stack/packages/py_xnrl}/package.py (97%) rename {spack-ext/repos/spack-stack => repos/spack_stack/spack_repo/spack_stack}/packages/r2d2/package.py (93%) rename {spack-ext/repos/spack-stack => repos/spack_stack/spack_repo/spack_stack}/packages/shumlib/package.py (98%) rename {spack-ext/repos/spack-stack/packages/soca-env => repos/spack_stack/spack_repo/spack_stack/packages/soca_env}/package.py (90%) rename {spack-ext/repos/spack-stack => repos/spack_stack/spack_repo/spack_stack}/packages/solo/package.py (92%) rename {spack-ext/repos/spack-stack/packages/ufs-pyenv => repos/spack_stack/spack_repo/spack_stack/packages/ufs_pyenv}/package.py (92%) rename {spack-ext/repos/spack-stack/packages/ufs-srw-app-env => repos/spack_stack/spack_repo/spack_stack/packages/ufs_srw_app_env}/package.py (95%) rename {spack-ext/repos/spack-stack/packages/ufs-utils-env => repos/spack_stack/spack_repo/spack_stack/packages/ufs_utils_env}/package.py (95%) rename {spack-ext/repos/spack-stack/packages/ufs-weather-model-env => repos/spack_stack/spack_repo/spack_stack/packages/ufs_weather_model_env}/package.py (95%) rename {spack-ext/repos/spack-stack/packages/upp-env => repos/spack_stack/spack_repo/spack_stack/packages/upp_env}/package.py (93%) rename {spack-ext/repos/spack-stack/packages/ww3-env => repos/spack_stack/spack_repo/spack_stack/packages/ww3_env}/package.py (94%) create mode 100644 repos/spack_stack/spack_repo/spack_stack/repo.yaml delete mode 100644 spack-ext/repos/spack-stack/repo.yaml diff --git a/.gitmodules b/.gitmodules index a69f3e4f1..1e0502061 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,8 +1,14 @@ [submodule "spack"] path = spack - url = https://github.com/jcsda/spack - branch = spack-stack-dev + #url = https://github.com/jcsda/spack + #branch = spack-stack-dev + url = https://github.com/climbfuji/spack + branch = feature/update_from_spack_dev_3b34f964da [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules branch = v1.3.0 +[submodule "repos/builtin"] + path = repos/builtin + url = https://github.com/jcsda/spack-packages + branch = spack-stack-dev diff --git a/repos/builtin b/repos/builtin new file mode 160000 index 000000000..063ada4e7 --- /dev/null +++ b/repos/builtin @@ -0,0 +1 @@ +Subproject commit 063ada4e7bbbfa6679b4661a97b9711b11a20577 diff --git a/spack-ext/repos/spack-stack/packages/ai-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/ai_env/package.py similarity index 91% rename from spack-ext/repos/spack-stack/packages/ai-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/ai_env/package.py index 1d50e0a78..902d6ddfb 100644 --- a/spack-ext/repos/spack-stack/packages/ai-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/ai_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/base-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/base_env/package.py similarity index 95% rename from spack-ext/repos/spack-stack/packages/base-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/base_env/package.py index 7307b24b8..44726f643 100644 --- a/spack-ext/repos/spack-stack/packages/base-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/base_env/package.py @@ -5,6 +5,7 @@ import sys +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/emc-gfs-wafs-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/emc_gfs_wafs_env/package.py similarity index 92% rename from spack-ext/repos/spack-stack/packages/emc-gfs-wafs-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/emc_gfs_wafs_env/package.py index f21738ec6..3e9f2f305 100644 --- a/spack-ext/repos/spack-stack/packages/emc-gfs-wafs-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/emc_gfs_wafs_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/emc-verif-global-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/emc_verif_global_env/package.py similarity index 94% rename from spack-ext/repos/spack-stack/packages/emc-verif-global-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/emc_verif_global_env/package.py index 932568f79..0aa08e905 100644 --- a/spack-ext/repos/spack-stack/packages/emc-verif-global-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/emc_verif_global_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/ewok/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/ewok/package.py similarity index 94% rename from spack-ext/repos/spack-stack/packages/ewok/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/ewok/package.py index 7f7bca138..dbb1bea5d 100644 --- a/spack-ext/repos/spack-stack/packages/ewok/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/ewok/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.python import PythonPackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/ewok-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/ewok_env/package.py similarity index 97% rename from spack-ext/repos/spack-stack/packages/ewok-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/ewok_env/package.py index f4d494132..3573828cf 100644 --- a/spack-ext/repos/spack-stack/packages/ewok-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/ewok_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/geos-gcm-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py similarity index 94% rename from spack-ext/repos/spack-stack/packages/geos-gcm-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py index 5c9515bb6..378bfbf01 100644 --- a/spack-ext/repos/spack-stack/packages/geos-gcm-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/gldas-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/gldas_env/package.py similarity index 91% rename from spack-ext/repos/spack-stack/packages/gldas-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/gldas_env/package.py index 1f41ede5e..d8e4fbf07 100644 --- a/spack-ext/repos/spack-stack/packages/gldas-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/gldas_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/global-workflow-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/global_workflow_env/package.py similarity index 95% rename from spack-ext/repos/spack-stack/packages/global-workflow-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/global_workflow_env/package.py index bfb4bc708..b2d422d43 100644 --- a/spack-ext/repos/spack-stack/packages/global-workflow-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/global_workflow_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/gmao-swell-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/gmao_swell_env/package.py similarity index 97% rename from spack-ext/repos/spack-stack/packages/gmao-swell-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/gmao_swell_env/package.py index 659ab9e5e..0ff0c3619 100644 --- a/spack-ext/repos/spack-stack/packages/gmao-swell-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/gmao_swell_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/gsi-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/gsi_env/package.py similarity index 93% rename from spack-ext/repos/spack-stack/packages/gsi-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/gsi_env/package.py index c5b47c644..30a11999b 100644 --- a/spack-ext/repos/spack-stack/packages/gsi-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/gsi_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/jedi-base-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env/package.py similarity index 97% rename from spack-ext/repos/spack-stack/packages/jedi-base-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env/package.py index eded9566a..08317d920 100644 --- a/spack-ext/repos/spack-stack/packages/jedi-base-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/jedi-fv3-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_fv3_env/package.py similarity index 90% rename from spack-ext/repos/spack-stack/packages/jedi-fv3-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/jedi_fv3_env/package.py index a9b1f971c..c02be8f42 100644 --- a/spack-ext/repos/spack-stack/packages/jedi-fv3-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_fv3_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/jedi-geos-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_geos_env/package.py similarity index 93% rename from spack-ext/repos/spack-stack/packages/jedi-geos-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/jedi_geos_env/package.py index 0820f7cb5..6473298f4 100644 --- a/spack-ext/repos/spack-stack/packages/jedi-geos-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_geos_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/jedi-mpas-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_mpas_env/package.py similarity index 92% rename from spack-ext/repos/spack-stack/packages/jedi-mpas-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/jedi_mpas_env/package.py index 2806fac83..38685d266 100644 --- a/spack-ext/repos/spack-stack/packages/jedi-mpas-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_mpas_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/jedi-neptune-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_neptune_env/package.py similarity index 91% rename from spack-ext/repos/spack-stack/packages/jedi-neptune-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/jedi_neptune_env/package.py index 871a76004..93708626c 100644 --- a/spack-ext/repos/spack-stack/packages/jedi-neptune-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_neptune_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/jedi-tools-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_tools_env/package.py similarity index 95% rename from spack-ext/repos/spack-stack/packages/jedi-tools-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/jedi_tools_env/package.py index a373bbe3c..0697f6618 100644 --- a/spack-ext/repos/spack-stack/packages/jedi-tools-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_tools_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/jedi-ufs-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_ufs_env/package.py similarity index 93% rename from spack-ext/repos/spack-stack/packages/jedi-ufs-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/jedi_ufs_env/package.py index b96372c83..783f196a6 100644 --- a/spack-ext/repos/spack-stack/packages/jedi-ufs-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_ufs_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/jedi-um-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_um_env/package.py similarity index 92% rename from spack-ext/repos/spack-stack/packages/jedi-um-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/jedi_um_env/package.py index b53b9e76f..a04f74e51 100644 --- a/spack-ext/repos/spack-stack/packages/jedi-um-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_um_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/nceplibs-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/nceplibs_env/package.py similarity index 94% rename from spack-ext/repos/spack-stack/packages/nceplibs-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/nceplibs_env/package.py index e5c25bfcb..ebadba8d5 100644 --- a/spack-ext/repos/spack-stack/packages/nceplibs-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/nceplibs_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py similarity index 96% rename from spack-ext/repos/spack-stack/packages/neptune-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py index 8da1eea86..a7ee62d0f 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py @@ -5,6 +5,7 @@ import sys +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py similarity index 95% rename from spack-ext/repos/spack-stack/packages/neptune-python-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py index 357442060..605ac0b59 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py @@ -5,6 +5,7 @@ import sys +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/py-xnrl/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/py_xnrl/package.py similarity index 97% rename from spack-ext/repos/spack-stack/packages/py-xnrl/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/py_xnrl/package.py index b66c5ec6c..63bd37b8f 100644 --- a/spack-ext/repos/spack-stack/packages/py-xnrl/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/py_xnrl/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.python import PythonPackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/r2d2/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/r2d2/package.py similarity index 93% rename from spack-ext/repos/spack-stack/packages/r2d2/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/r2d2/package.py index 268920e17..14de5150b 100644 --- a/spack-ext/repos/spack-stack/packages/r2d2/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/r2d2/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.python import PythonPackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/shumlib/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/shumlib/package.py similarity index 98% rename from spack-ext/repos/spack-stack/packages/shumlib/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/shumlib/package.py index 4e830afcf..64ba54bd6 100644 --- a/spack-ext/repos/spack-stack/packages/shumlib/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/shumlib/package.py @@ -6,6 +6,7 @@ import os import subprocess +from spack_repo.builtin.build_systems.makefile import MakefilePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/soca-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/soca_env/package.py similarity index 90% rename from spack-ext/repos/spack-stack/packages/soca-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/soca_env/package.py index 51be48aeb..a730a72a1 100644 --- a/spack-ext/repos/spack-stack/packages/soca-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/soca_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/solo/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/solo/package.py similarity index 92% rename from spack-ext/repos/spack-stack/packages/solo/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/solo/package.py index 41c99e6a8..f69fbd644 100644 --- a/spack-ext/repos/spack-stack/packages/solo/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/solo/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.python import PythonPackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/ufs-pyenv/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/ufs_pyenv/package.py similarity index 92% rename from spack-ext/repos/spack-stack/packages/ufs-pyenv/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/ufs_pyenv/package.py index 3540c4809..85234175d 100644 --- a/spack-ext/repos/spack-stack/packages/ufs-pyenv/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/ufs_pyenv/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/ufs-srw-app-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/ufs_srw_app_env/package.py similarity index 95% rename from spack-ext/repos/spack-stack/packages/ufs-srw-app-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/ufs_srw_app_env/package.py index 8233eae8f..4afc2b015 100644 --- a/spack-ext/repos/spack-stack/packages/ufs-srw-app-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/ufs_srw_app_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/ufs-utils-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/ufs_utils_env/package.py similarity index 95% rename from spack-ext/repos/spack-stack/packages/ufs-utils-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/ufs_utils_env/package.py index 26f02c38e..d4e408458 100644 --- a/spack-ext/repos/spack-stack/packages/ufs-utils-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/ufs_utils_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/ufs-weather-model-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/ufs_weather_model_env/package.py similarity index 95% rename from spack-ext/repos/spack-stack/packages/ufs-weather-model-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/ufs_weather_model_env/package.py index 1d309362d..07c8c36a6 100644 --- a/spack-ext/repos/spack-stack/packages/ufs-weather-model-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/ufs_weather_model_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/upp-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/upp_env/package.py similarity index 93% rename from spack-ext/repos/spack-stack/packages/upp-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/upp_env/package.py index 77325bb5f..1d24ad22e 100644 --- a/spack-ext/repos/spack-stack/packages/upp-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/upp_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/spack-ext/repos/spack-stack/packages/ww3-env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/ww3_env/package.py similarity index 94% rename from spack-ext/repos/spack-stack/packages/ww3-env/package.py rename to repos/spack_stack/spack_repo/spack_stack/packages/ww3_env/package.py index 588fa5a5e..1e0345253 100644 --- a/spack-ext/repos/spack-stack/packages/ww3-env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/ww3_env/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.bundle import BundlePackage from spack.package import * diff --git a/repos/spack_stack/spack_repo/spack_stack/repo.yaml b/repos/spack_stack/spack_repo/spack_stack/repo.yaml new file mode 100644 index 000000000..ef1e56073 --- /dev/null +++ b/repos/spack_stack/spack_repo/spack_stack/repo.yaml @@ -0,0 +1,3 @@ +repo: + namespace: spack_stack + api: v2.0 diff --git a/spack-ext/repos/spack-stack/repo.yaml b/spack-ext/repos/spack-stack/repo.yaml deleted file mode 100644 index 9d56860ff..000000000 --- a/spack-ext/repos/spack-stack/repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: spack-stack From 0740e30bfe64f770dde8c9e3a2a7b62e9b220b5a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Jul 2025 09:04:11 -0600 Subject: [PATCH 002/163] Remove logic to register repos and extensions from setup.sh, update submodule pointer for spack --- .gitmodules | 6 ++--- setup.sh | 63 ++++++++++++++++++++++++++++------------------------- spack | 2 +- 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/.gitmodules b/.gitmodules index 1e0502061..96db55efe 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,6 +9,6 @@ url = https://github.com/noaa-emc/cmakemodules branch = v1.3.0 [submodule "repos/builtin"] - path = repos/builtin - url = https://github.com/jcsda/spack-packages - branch = spack-stack-dev + path = repos/builtin + url = https://github.com/jcsda/spack-packages + branch = spack-stack-dev diff --git a/setup.sh b/setup.sh index c93e7578e..6c355bf0b 100755 --- a/setup.sh +++ b/setup.sh @@ -12,41 +12,44 @@ esac export SPACK_STACK_DIR echo "Setting environment variable SPACK_STACK_DIR to ${SPACK_STACK_DIR}" -source ${SPACK_STACK_DIR:?}/spack/share/spack/setup-env.sh +# Avoid using ~/.spack directory for caching, spack repo clones, etc. +export SPACK_DISABLE_LOCAL_CONFIG=true +export SPACK_USER_CACHE_PATH=${SPACK_STACK_DIR}/cache/user_cache + +source ${SPACK_STACK_DIR}/spack/share/spack/setup-env.sh echo "Sourcing spack environment ${SPACK_STACK_DIR}/spack/share/spack/setup-env.sh" -# Avoid using ~/.spack directory for caching, bootstrap, etc. -export SPACK_DISABLE_LOCAL_CONFIG=true -export SPACK_USER_CACHE_PATH=$SPACK_ROOT/user_cache +# Avoid using ~/.spack directory for bootstraping echo "Changing bootstrap path to $(spack bootstrap root '$spack/bootstrap')" # Get the current hash of the spack-stack code export SPACK_STACK_HASH=`git rev-parse --short HEAD` echo "Current hash of spack-stack is ${SPACK_STACK_HASH}" -# Register the spack extension -libpath=${SPACK_STACK_DIR}/spack-ext/lib/jcsda-emc/spack-stack -if [ -d ${libpath}/stack/cmd ]; then - spack config --scope defaults add "config:extensions:$libpath" -else - echo "FATAL ERROR: Could not find spack-stack extensions in ${libpath}" - return 1 -fi - -# Register the jcsda-emc repos -msg1="Added repo with namespace" -msg2="Repository is already registered with Spack" -for repo in spack-stack; do - repodir=${SPACK_STACK_DIR}/spack-ext/repos/$repo - othererrors=$( ( spack repo add $repodir --scope defaults 2>&1 | grep -v -e "$msg1" -e "$msg2" ) || true ) - if [ $(echo "$othererrors" | grep -c .) -ne 0 ]; then - echo "$othererrors" - return 2 - fi - if [ ! -d $repodir ]; then - echo "FATAL ERROR: Repo directory $repodir does not exist!" - return 3 - fi -done - -echo "spack-stack extensions and repos are registered" +## Register the spack extension +#libpath=${SPACK_STACK_DIR}/spack-ext/lib/jcsda-emc/spack-stack +#if [ -d ${libpath}/stack/cmd ]; then +# spack config --scope defaults add "config:extensions:$libpath" +#else +# echo "FATAL ERROR: Could not find spack-stack extensions in ${libpath}" +# return 1 +#fi +# +### Register the jcsda-emc repos +##msg1="Added repo with namespace" +##msg2="Repository is already registered with Spack" +##for repo in spack-stack; do +## repodir=${SPACK_STACK_DIR}/spack-ext/repos/$repo +## othererrors=$( ( spack repo add $repodir --scope defaults 2>&1 | grep -v -e "$msg1" -e "$msg2" ) || true ) +## if [ $(echo "$othererrors" | grep -c .) -ne 0 ]; then +## echo "$othererrors" +## return 2 +## fi +## if [ ! -d $repodir ]; then +## echo "FATAL ERROR: Repo directory $repodir does not exist!" +## return 3 +## fi +##done +# +##echo "spack-stack extensions and repos are registered" +#echo "spack-stack extensions are registered" diff --git a/spack b/spack index 43381fe76..cf9860864 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 43381fe761f7280ac91fbdda462f86a062979843 +Subproject commit cf9860864bbebca7f653c8d0ce0b558c772152a4 From 919f01c5c7b998deb91a3fa66bcfa9947c493f70 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Jul 2025 09:39:53 -0600 Subject: [PATCH 003/163] Update .gitmodules and submodule pointer for spack --- .gitmodules | 2 +- spack | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 96db55efe..121b13a3d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ #url = https://github.com/jcsda/spack #branch = spack-stack-dev url = https://github.com/climbfuji/spack - branch = feature/update_from_spack_dev_3b34f964da + branch = feature/update_from_spack_dev_53bedfb551 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index cf9860864..da59344d8 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit cf9860864bbebca7f653c8d0ce0b558c772152a4 +Subproject commit da59344d8677ed3cbdcf1dee865b6855d977c84e From 383b63db56aae959137712278ea67f55754af0cd Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 23 Jul 2025 05:58:41 -0600 Subject: [PATCH 004/163] Update submodule pointers for repos/builtin and spack --- .gitmodules | 2 +- .../tier2/blackpearl/{compilers.yaml => compilers.yaml.old} | 0 repos/builtin | 2 +- spack | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename configs/sites/tier2/blackpearl/{compilers.yaml => compilers.yaml.old} (100%) diff --git a/.gitmodules b/.gitmodules index 121b13a3d..60aa318b7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ #url = https://github.com/jcsda/spack #branch = spack-stack-dev url = https://github.com/climbfuji/spack - branch = feature/update_from_spack_dev_53bedfb551 + branch = feature/update_to_spack_v1p0p0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/configs/sites/tier2/blackpearl/compilers.yaml b/configs/sites/tier2/blackpearl/compilers.yaml.old similarity index 100% rename from configs/sites/tier2/blackpearl/compilers.yaml rename to configs/sites/tier2/blackpearl/compilers.yaml.old diff --git a/repos/builtin b/repos/builtin index 063ada4e7..beafe0546 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 063ada4e7bbbfa6679b4661a97b9711b11a20577 +Subproject commit beafe0546edf678a526ce697c701e1756b2f86c6 diff --git a/spack b/spack index da59344d8..1e404d9ec 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit da59344d8677ed3cbdcf1dee865b6855d977c84e +Subproject commit 1e404d9ec16b1a58e0352484912f63a8918fd36d From 02e3cee7352ab6a642bf9d85bed079735a7e7191 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 23 Jul 2025 06:01:27 -0600 Subject: [PATCH 005/163] configs/common/packages.yaml: avoid using rust-boostrap, switch to zlib instead of zlib-ng for zlib-api --- configs/common/packages.yaml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 330808ce5..723fff409 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -7,7 +7,7 @@ packages: jpeg: [libjpeg-turbo] pkgconfig: [pkg-config] yacc: [bison] - zlib-api: [zlib-ng] + zlib-api: [zlib] # awscli-v2: require: '~examples' @@ -231,11 +231,6 @@ packages: require: '@3.11.11 ~crypt' py-cartopy: require: '+plotting' - # Avoid full rust dependency: version 43+ requires py-maturin - # DH* 20250214 No longer needed? - # https://github.com/JCSDA/spack-stack/pull/1519 - py-cryptography: - require: '@:42 +rust_bootstrap' # Pin py-cython to avoid duplicate packages py-cython: require: '@3' @@ -254,8 +249,6 @@ packages: # https://github.com/JCSDA/spack-stack/issues/1276 py-matplotlib: require: '@3.7.4' - py-maturin: - require: '+rust_bootstrap' # https://github.com/Unidata/netcdf4-python/issues/1389 # https://github.com/Unidata/netcdf4-python/issues/1389#issuecomment-2599760051 py-netcdf4: @@ -264,16 +257,12 @@ packages: py-numpy: require: - '@1.26' - py-rpds-py: - require: '+rust_bootstrap' # To avoid duplicate packages py-ruamel-yaml: require: '@0.17.16' # Pin the py-setuptools version to avoid duplicate Python packages py-setuptools: require: '@69' - py-setuptools-rust: - require: '+rust_bootstrap' # When making changes here, also check the packages.yaml for atlantis, nautilus, blackpearl py-torch: require: From d356c731417b12527cf6922e19f209b918828e0f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 25 Jul 2025 15:09:57 -0600 Subject: [PATCH 006/163] Saving progress --- configs/common/packages.yaml | 4 +- configs/common/packages_gcc.yaml | 11 +++++ configs/common/packages_oneapi.yaml | 36 +++++++++++++--- configs/sites/tier2/blackpearl/packages.yaml | 15 ++++--- .../sites/tier2/blackpearl/packages_aocc.yaml | 5 --- .../tier2/blackpearl/packages_gcc-13.2.1.yaml | 17 ++++++++ .../sites/tier2/blackpearl/packages_gcc.yaml | 5 --- .../blackpearl/packages_oneapi-2024.2.1.yaml | 43 +++++++++++++++++++ .../blackpearl/packages_oneapi-2025.1.0.yaml | 43 +++++++++++++++++++ .../tier2/blackpearl/packages_oneapi.yaml | 25 ----------- .../jcsda-emc/spack-stack/stack/stack_env.py | 27 +++++++++--- 11 files changed, 175 insertions(+), 56 deletions(-) delete mode 100644 configs/sites/tier2/blackpearl/packages_aocc.yaml create mode 100644 configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml delete mode 100644 configs/sites/tier2/blackpearl/packages_gcc.yaml create mode 100644 configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml create mode 100644 configs/sites/tier2/blackpearl/packages_oneapi-2025.1.0.yaml delete mode 100644 configs/sites/tier2/blackpearl/packages_oneapi.yaml diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 723fff409..096f4e799 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -143,7 +143,9 @@ packages: hdf: require: '@4.2.15 +external-xdr ~fortran ~netcdf' hdf5: - require: '@1.14.5 +hl +fortran +mpi +threadsafe ~szip' + require: + - '@1.14.5' + - '+hl +fortran +mpi +threadsafe ~szip' # Newer versions of hdf-eos2 require manual downloading, avoid hdf-eos2: require: '@2.20v1.00' diff --git a/configs/common/packages_gcc.yaml b/configs/common/packages_gcc.yaml index 70012d1a0..34b78df4f 100644 --- a/configs/common/packages_gcc.yaml +++ b/configs/common/packages_gcc.yaml @@ -1,5 +1,16 @@ packages: + c: + require: + - "gcc" + cxx: + require: + - "gcc" + fortran: + require: + - "gcc" all: + require: + - '%[when=%c]gcc %[when=%cxx]gcc %[when=%fortran]gcc' providers: blas: [openblas] fftw-api: [fftw] diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index 47f8df097..bfe05acab 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -1,12 +1,27 @@ packages: + # Allow certain packages to be built with GCC as long as they only use + # C and/or C++. Because Fortran compilers are not ABI compatible, we + # need to force spack to use one and only one compiler (Intel oneAPI) + c: + require: + - one_of: ["intel-oneapi-compilers", "gcc"] + prefer: + - "intel-oneapi-compilers" + cxx: + require: + - one_of: ["intel-oneapi-compilers", "gcc"] + prefer: + - "intel-oneapi-compilers" + fortran: + require: + - "intel-oneapi-compilers" all: + require: + - '%[when=%c]oneapi %[when=%cxx]oneapi %[when=%fortran]oneapi' providers: blas: [intel-oneapi-mkl] fftw-api: [intel-oneapi-mkl] lapack: [intel-oneapi-mkl] - cmake: - require: - - '%gcc' ectrans: require: - '+mkl ~fftw' @@ -17,13 +32,20 @@ packages: require: - '^intel-oneapi-mkl' bison: - require: '%gcc' + require: + - '%gcc' + cmake: + require: + - '%gcc' gmake: - require: '%gcc' + require: + - '%gcc' libmd: - require: '%gcc' + require: + - '%gcc' libbsd: - require: '%gcc' + require: + - '%gcc' met: require: - '+shared-intel' diff --git a/configs/sites/tier2/blackpearl/packages.yaml b/configs/sites/tier2/blackpearl/packages.yaml index e5e13f46a..495587560 100644 --- a/configs/sites/tier2/blackpearl/packages.yaml +++ b/configs/sites/tier2/blackpearl/packages.yaml @@ -1,10 +1,12 @@ packages: # Modification of common packages - + # Prevent building zlib, zlib-ng + zlib-api: + buildable: false # Pin py-torch to 2.5.1 to avoid build errors with 2.6.0 py-torch: require: - - '@2.5.1' + - '@2.5.1' # All other packages listed alphabetically autoconf: @@ -55,10 +57,6 @@ packages: externals: - spec: git-lfs@3.4.1 prefix: /usr - #gmake: - # externals: - # - spec: gmake@4.3 - # prefix: /usr grep: externals: - spec: grep@3.6 @@ -68,6 +66,7 @@ packages: - spec: groff@1.22.4 prefix: /usr m4: + # DH* NEEDED? buildable: false externals: - spec: m4@1.4.19 prefix: /usr @@ -91,3 +90,7 @@ packages: externals: - spec: wget@1.21.1 prefix: /usr + zlib: + externals: + - spec: zlib@1.2.11 + prefix: /usr diff --git a/configs/sites/tier2/blackpearl/packages_aocc.yaml b/configs/sites/tier2/blackpearl/packages_aocc.yaml deleted file mode 100644 index 3e690c586..000000000 --- a/configs/sites/tier2/blackpearl/packages_aocc.yaml +++ /dev/null @@ -1,5 +0,0 @@ -packages: - all: - compiler:: [aocc@4.2.0,gcc@11.4.1] - providers: - mpi:: [openmpi@5.0.5] diff --git a/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml b/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml new file mode 100644 index 000000000..f5b2423cd --- /dev/null +++ b/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml @@ -0,0 +1,17 @@ +packages: + mpi: + require: + - "openmpi@=5.0.6" + gcc: + externals: + - spec: gcc@13.2.1 languages:='c,c++,fortran' + prefix: /opt/rh/gcc-toolset-13/root/usr + extra_attributes: + compilers: + c: /opt/rh/gcc-toolset-13/root/usr/bin/gcc + cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ + fortran: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@13.2.1%gcc@13.2.1 + prefix: /opt/rh/gcc-toolset-13/root/usr diff --git a/configs/sites/tier2/blackpearl/packages_gcc.yaml b/configs/sites/tier2/blackpearl/packages_gcc.yaml deleted file mode 100644 index 730601342..000000000 --- a/configs/sites/tier2/blackpearl/packages_gcc.yaml +++ /dev/null @@ -1,5 +0,0 @@ -packages: - all: - compiler:: [gcc@13.3.0] - providers: - mpi:: [openmpi@5.0.5] diff --git a/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml b/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml new file mode 100644 index 000000000..6f0e5a23a --- /dev/null +++ b/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml @@ -0,0 +1,43 @@ +packages: + mpi: + buildable: False + require: + - "intel-oneapi-mpi@2021.13" + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2024.2.1 + prefix: /opt/intel/oneapi + extra_attributes: + compilers: + c: /opt/intel/oneapi/2024.2/bin/icx + cxx: /opt/intel/oneapi/2024.2/bin/icpx + fortran: /opt/intel/oneapi/2024.2/bin/ifort + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.13 + prefix: /opt/intel/oneapi + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2 + prefix: /opt/intel/oneapi + intel-oneapi-runtime: + # LINK TO BUG REPORT HERE + #buildable: false + externals: + - spec: intel-oneapi-runtime@2024.2.1 + prefix: /opt/intel/oneapi + gcc: + externals: + - spec: gcc@11.4.1 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@11.4.1%gcc@11.4.1 + prefix: /usr diff --git a/configs/sites/tier2/blackpearl/packages_oneapi-2025.1.0.yaml b/configs/sites/tier2/blackpearl/packages_oneapi-2025.1.0.yaml new file mode 100644 index 000000000..ab3e0217d --- /dev/null +++ b/configs/sites/tier2/blackpearl/packages_oneapi-2025.1.0.yaml @@ -0,0 +1,43 @@ +packages: + mpi: + buildable: False + require: + - "intel-oneapi-mpi@2021.15" + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2025.1.0 + prefix: /opt/intel/oneapi + extra_attributes: + compilers: + c: /opt/intel/oneapi/2025.1/bin/icx + cxx: /opt/intel/oneapi/2025.1/bin/icpx + fortran: /opt/intel/oneapi/2025.1/bin/ifx + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.15 + prefix: /opt/intel/oneapi + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.1 + prefix: /opt/intel/oneapi + intel-oneapi-runtime: + # LINK TO BUG REPORT HERE + #buildable: false + externals: + - spec: intel-oneapi-runtime@2025.1.0 + prefix: /opt/intel/oneapi + gcc: + externals: + - spec: gcc@11.4.1 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@11.4.1%gcc@11.4.1 + prefix: /usr diff --git a/configs/sites/tier2/blackpearl/packages_oneapi.yaml b/configs/sites/tier2/blackpearl/packages_oneapi.yaml deleted file mode 100644 index d79986c4e..000000000 --- a/configs/sites/tier2/blackpearl/packages_oneapi.yaml +++ /dev/null @@ -1,25 +0,0 @@ -packages: - all: - compiler:: [oneapi@2024.2.1, oneapi@2025.1.0, gcc@11.4.1] - providers: - mpi:: [intel-oneapi-mpi@2021.13, intel-oneapi-mpi@2021.15] - mpi: - buildable: False - intel-oneapi-mpi: - externals: - - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 - prefix: /opt/intel/oneapi - - spec: intel-oneapi-mpi@2021.15%oneapi@2025.1.0 - prefix: /opt/intel/oneapi - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1 - prefix: /opt/intel/oneapi - - spec: intel-oneapi-mkl@2025.1%oneapi@2025.1.0 - prefix: /opt/intel/oneapi - intel-oneapi-runtime: - externals: - - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 - prefix: /opt/intel/oneapi - - spec: intel-oneapi-runtime@2025.1.0%oneapi@2025.1.0 - prefix: /opt/intel/oneapi diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index aa95f3bf8..df5857e0f 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -171,7 +171,11 @@ def _copy_common_includes(self): self._copy_or_merge_includes("modules", modules_yaml_path, modules_yaml_modulesys_path, destination) # Merge or copy common package config(s) packages_yaml_path = os.path.join(common_path, "packages.yaml") - packages_compiler_yaml_path = os.path.join(common_path, f"packages_{self.compiler.split('@')[0]}.yaml") + if not "@" in self.compiler: + raise Exception("Invalid value for self.compiler, '@' not found. " + \ + "Expected: 'name@version' or 'name@=version'") + compiler_name, compiler_version = self.compiler.replace("@=", "@").split('@') + packages_compiler_yaml_path = os.path.join(common_path, f"packages_{compiler_name}.yaml") destination = os.path.join(env_common_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) @@ -204,7 +208,13 @@ def _copy_site_includes(self): self._copy_or_merge_includes("modules", modules_yaml_path, modules_yaml_modulesys_path, destination) # Merge or copy site package config(s) packages_yaml_path = os.path.join(env_path, "packages.yaml") - packages_compiler_yaml_path = os.path.join(env_path, f"packages_{self.compiler.split('@')[0]}.yaml") + if not "@" in self.compiler: + raise Exception("Invalid value for self.compiler, '@' not found. " + \ + "Expected: 'name@version' or 'name@=version'") + compiler_name, compiler_version = self.compiler.replace("@=", "@").split('@') + packages_compiler_yaml_path = os.path.join(env_path, f"packages_{compiler_name}-{compiler_version}.yaml") + if not os.path.exists(packages_compiler_yaml_path): + raise Exception(f"File {packages_compiler_yaml_path} not found. ") destination = os.path.join(env_site_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) @@ -249,9 +259,12 @@ def write(self): # Copy site config first so that it takes precedence in env if self.site != "none": self._copy_site_includes() - # By default remove all other versions of the specified compiler - if not self.keepallcompilers and "@" in self.compiler: - self.filter_site_compilers(os.path.join(env_dir, "site", "compilers.yaml"), self.compiler) + # DH* + print("WARNING, filter_site_compilers currently commented out") + ## By default remove all other versions of the specified compiler + #if not self.keepallcompilers and "@" in self.compiler: + # self.filter_site_compilers(os.path.join(env_dir, "site", "compilers.yaml"), self.compiler) + # *DH # Copy common include files self._copy_common_includes() @@ -283,8 +296,8 @@ def write(self): original_sections[key] = copy.deepcopy(section) # Commonly used config settings - compiler = f"packages:all:prefer:['%{self.compiler}']" - spack.config.add(compiler, scope=env_scope) + #compiler = f"packages:all:prefer:['%{self.compiler}']" + #spack.config.add(compiler, scope=env_scope) # Also update compiler definitions if matrices are used # DH I am too stupid to do this the "spack way" ... definitions = spack.config.get("definitions", scope=env_scope) From 7254b6b8951a760d669928f069442ffbffdecd0e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 30 Jul 2025 13:11:10 -0600 Subject: [PATCH 007/163] Save progress for configs/** --- configs/common/modules_lmod.yaml | 9 +- configs/common/modules_tcl.yaml | 9 +- configs/common/packages.yaml | 233 +++++++++++++------- configs/common/packages_intel.yaml | 6 + configs/common/packages_oneapi.yaml | 6 + configs/sites/tier2/blackpearl/modules.yaml | 3 +- 6 files changed, 176 insertions(+), 90 deletions(-) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index b6acf37fa..fbbb75773 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -87,14 +87,13 @@ modules: - xtrans - xz - yaksa - # Do not create MPI modules - we create our own meta modules + # By default, do not create MPI modules - cray-mpich - - mpich - - openmpi - intel-mpi - intel-oneapi-mpi - # Do not create python modules - we create our own meta modules - - python + - mpich + - mpt + - openmpi hash_length: 0 all: autoload: direct diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index fea56192b..c6cae9461 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -85,14 +85,13 @@ modules: - xtrans - xz - yaksa - # Do not create MPI modules - we create our own meta modules + # By default, do not create MPI modules - cray-mpich - - mpich - - openmpi - intel-mpi - intel-oneapi-mpi - # Do not create python modules - we create our own meta modules - - python + - mpich + - mpt + - openmpi hash_length: 0 all: autoload: direct diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 096f4e799..ea841dd9a 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -10,19 +10,24 @@ packages: zlib-api: [zlib] # awscli-v2: - require: '~examples' + require: + - '~examples' bacio: - require: '@2.6.0' + require: + - '@2.6.0' bison: - require: '@3.4:' + require: + - '@3.4:' # 1.85 incompatible with ecflow@5.11.4 - use latest 'good version' boost: require: - '@1.84 ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden' bufr: - require: '@12.1.0 +python' + require: + - '@12.1.0 +python' bufr-query: - require: '@0.0.4 +python' + require: + - '@0.0.4 +python' cairo: require: '+pic' cdo: @@ -45,9 +50,11 @@ packages: - '+ownlibs' # Attention - when updating also check the various jcsda-emc-bundles env packages crtm: - require: '+fix' + require: + - '+fix' ecbuild: - require: '@3.7.2' + require: + - '@3.7.2' eccodes: require: - '@2.33.0' @@ -56,14 +63,17 @@ packages: require: - '@5.11.4 +ui' eckit: - require: '@1.28.3 linalg=eigen,lapack compression=lz4,bzip2' + require: + - '@1.28.3 linalg=eigen,lapack compression=lz4,bzip2' ecmwf-atlas: - require: '@0.40.0 +fckit +trans +tesselation +fftw' + require: + - '@0.40.0 +fckit +trans +tesselation +fftw' ectrans: require: - '@1.5.0' eigen: - require: '@3.4.0' + require: + - '@3.4.0' # Attention - when updating the version also check the common modules.yaml # config and update the projections for lmod/tcl. # Also, check the acorn and derecho site configs which have esmf modifications. @@ -88,34 +98,44 @@ packages: require: - '~libbsd' fckit: - require: '@0.13.2 +eckit' + require: + - '@0.13.2 +eckit' fftw: - require: '@3.3.10' + require: + - '@3.3.10' fiat: - require: '@1.4.1' + require: + - '@1.4.1' flex: # Pin version to avoid duplicates - require: '@2.6.4' + require: + - '@2.6.4' fms: require: - '@2024.02' - 'precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io' fontconfig: - require: '+pic' + require: + - '+pic' freetype: - require: '+pic' + require: + - '+pic' g2: require: - '@3.5.1' g2c: - require: '@2.1.0' + require: + - '@2.1.0' g2tmpl: require: - '@1.13.0' - gcc-runtime: - require: '%gcc' + # DH* NEEDED??? + #gcc-runtime: + # require: + # - '%gcc' gfsio: - require: '@1.4.2' + require: + - '@1.4.2' #git-lfs: # Assume git-lfs is provided, hard to install # because of dependencies on go/go-bootstrap. @@ -132,86 +152,116 @@ packages: require: - one_of: ['@=4.2.1', '@=4.4.1'] grib-util: - require: '@1.4.0' + require: + - '@1.4.0' gsibec: require: - '@1.2.1' gsi-ncdiag: - require: '@1.1.2' + require: + - '@1.1.2' gsl-lite: - require: '@0.37.0' + require: + - '@0.37.0' hdf: - require: '@4.2.15 +external-xdr ~fortran ~netcdf' + require: + - '@4.2.15 +external-xdr ~fortran ~netcdf' hdf5: require: - '@1.14.5' - '+hl +fortran +mpi +threadsafe ~szip' # Newer versions of hdf-eos2 require manual downloading, avoid hdf-eos2: - require: '@2.20v1.00' + require: + - '@2.20v1.00' ip: - require: '@5.4.0 precision=4,d,8' + require: + - '@5.4.0 precision=4,d,8' ip2: - require: '@1.1.2' + require: + - '@1.1.2' jasper: - require: '@2.0.32' + require: + - '@2.0.32' jedi-cmake: - require: '@1.4.0' + require: + - '@1.4.0' landsfcutil: - require: '@2.4.2' + require: + - '@2.4.2' libjpeg-turbo: - require: '@2.1.0' + require: + - '@2.1.0' libpng: - require: '@1.6.37 +pic' + require: + - '@1.6.37 +pic' # libtirpc +gssapi doesn't build with oneapi@2025 # https://github.com/JCSDA/spack-stack/issues/1441 libtirpc: - require: '~gssapi' + require: + - '~gssapi' # Newest version of magics needed for oneapi compilers magics: - require: "@4.15.3:" + require: + - '@4.15.3:' mapl: - require: '@2.53 ~shared ~f2py' - variants: '+pflogger' + require: + - '@2.53 ~shared ~f2py +pflogger' + #variants: '+pflogger' met: require: - '@12.0.1' - '+python' - '+grib2' metplus: - require: '@6.0.0' + require: + - '@6.0.0' metis: - require: '+int64 +real64' + require: + - '+int64 +real64' mpich: - require: '~hwloc' + require: + - '~hwloc' nco: - require: '@5.2.4 ~doc' + require: + - '~doc' # ncview - when adding information here, also check Orion # and Discover site configs nemsio: - require: '@2.5.4' + require: + - '@2.5.4' nemsiogfs: - require: '@2.5.3' + require: + - '@2.5.3' nccmp: - require: '@1.9.0.1' + require: + - '@1.9.0.1' ncio: - require: '@1.1.2' + require: + - '@1.1.2' netcdf-c: # If using 4.9.1, turn off byterange variant to fix compile error: ~byterange - require: '@4.9.2 +dap +mpi ~parallel-netcdf ~szip +parallel_tests build_system=autotools' + require: + - '@4.9.2 +dap +mpi ~parallel-netcdf ~szip +parallel_tests build_system=autotools' netcdf-cxx4: - require: '@4.3.1' + require: + - '@4.3.1' netcdf-fortran: - require: '@4.6.1 +parallel_tests' + require: + - '@4.6.1 +parallel_tests' # ninja - when adding information here, also check Discover site config odc: - require: '@1.6.1 ~fortran' + require: + - '@1.6.1 ~fortran' openmpi: - require: '~internal-hwloc +two_level_namespace' + require: + - '~internal-hwloc +two_level_namespace' openssl: - require: '+shared' + require: + - '+shared' p4est: - require: '@2.8.7' + require: + - '@2.8.7' parallelio: require: - '@2.6.2' @@ -219,62 +269,78 @@ packages: require: - '@1.12.3' pflogger: - require: '+mpi' + require: + - '+mpi' pixman: - require: '+pic' + require: + - '+pic' prod-util: - require: '@2.1.1' + require: + - '@2.1.1' proj: - require: '~tiff' + require: + - '~tiff' # Turn off crypt variant for Python; this leads to build errors # with Intel in py-cryptography unless external curl and openssl # are removed, which itself is problematic. python: - require: '@3.11.11 ~crypt' + require: + - '@3.11 ~crypt' py-cartopy: - require: '+plotting' + require: + - '+plotting' # Pin py-cython to avoid duplicate packages py-cython: - require: '@3' + require: + - '@3' # As long as we need the "delayed" variant for py-dask, # we are restricted to version 2021.03.0 (last one that has it) # and newer than 2021.01.0 because of these issues: # https://github.com/JCSDA/spack-stack/issues/1216 # https://github.com/pydata/xarray/issues/8917 py-dask: - require: '@2021.3.0' + require: + - '@2021.3.0' # To avoid duplicate packages py-flit-core: - require: '@3.8.0' + require: + - '@3.8.0' py-h5py: - require: '~mpi' + require: + - '~mpi' # https://github.com/JCSDA/spack-stack/issues/1276 py-matplotlib: - require: '@3.7.4' + require: + - '@3.7.4' # https://github.com/Unidata/netcdf4-python/issues/1389 # https://github.com/Unidata/netcdf4-python/issues/1389#issuecomment-2599760051 py-netcdf4: - require: '@1.7.1 ~mpi' + require: + - '@1.7.1 ~mpi' # To avoid duplicate packages py-numpy: require: - '@1.26' # To avoid duplicate packages py-ruamel-yaml: - require: '@0.17.16' + require: + - '@0.17.16' # Pin the py-setuptools version to avoid duplicate Python packages py-setuptools: - require: '@69' + require: + - '@69' # When making changes here, also check the packages.yaml for atlantis, nautilus, blackpearl py-torch: require: - '+custom-protobuf ~mkldnn' # To avoid duplicate packages py-urllib3: - require: '@1.26.12' + require: + - '@1.26.12' # To avoid duplicate packages py-versioneer: - require: '@0.28' + require: + - '@0.28' qt: require: - '@5' @@ -282,30 +348,41 @@ packages: require: - '@7.0.4 +mpi+metis~shared~threads~mpi_thread+noarch+esmumps' sfcio: - require: '@1.4.2' + require: + - '@1.4.2' shumlib: - require: '@macos_clang_linux_intel_port' + require: + - '@macos_clang_linux_intel_port' sigio: - require: '@2.3.3' + require: + - '@2.3.3' sp: - require: '@2.5.0 precision=4,d,8' + require: + - '@2.5.0 precision=4,d,8' udunits: - require: '@2.2.28' + require: + - '@2.2.28' ufs-utils: - require: '@1.13.0' + require: + - '@1.13.0' # Note - we can remove upp from stack at some point? upp: - require: '@10.0.10' + require: + - '@10.0.10' # Need extradeps for grib-utils, enable by default to avoid duplicate packages w3emc: - require: '@2.10.0 precision=4,d,8 +extradeps' + require: + - '@2.10.0 precision=4,d,8 +extradeps' w3nco: - require: '@2.4.1' + require: + - '@2.4.1' # See macOS site config for wgrib2 version/variant overrides wgrib2: require: - '@3.6.0' wrf-io: - require: '@1.2.0' + require: + - '@1.2.0' zstd: - require: '@1.5.6 +programs' + require: + - '@1.5.6 +programs' diff --git a/configs/common/packages_intel.yaml b/configs/common/packages_intel.yaml index 0498f91d4..851c3fd64 100644 --- a/configs/common/packages_intel.yaml +++ b/configs/common/packages_intel.yaml @@ -4,6 +4,9 @@ packages: blas: [intel-oneapi-mkl] fftw-api: [intel-oneapi-mkl] lapack: [intel-oneapi-mkl] + antlr: + require: + - '%gcc' cmake: require: - '%gcc' @@ -42,3 +45,6 @@ packages: pixman: require: - '%gcc' + nco: + require: + - '%gcc' \ No newline at end of file diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index bfe05acab..49dccd083 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -31,6 +31,9 @@ packages: py-numpy: require: - '^intel-oneapi-mkl' + antlr: + require: + - '%gcc' bison: require: - '%gcc' @@ -49,6 +52,9 @@ packages: met: require: - '+shared-intel' + nco: + require: + - '%gcc' parallel-netcdf: require: - '+shared-intel' diff --git a/configs/sites/tier2/blackpearl/modules.yaml b/configs/sites/tier2/blackpearl/modules.yaml index 57e6be9fe..43f631874 100644 --- a/configs/sites/tier2/blackpearl/modules.yaml +++ b/configs/sites/tier2/blackpearl/modules.yaml @@ -5,6 +5,5 @@ modules: tcl: include: # List of packages for which we need modules that are blacklisted by default + - intel-oneapi-mpi - openmpi - - mpich - - python From e355a02e6aea3d5176975de5e0ed9b946dd89d81 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 30 Jul 2025 13:11:41 -0600 Subject: [PATCH 008/163] Update spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py and associated templates --- .../spack-stack/stack/meta_modules.py | 1028 ++++++----------- .../stack/meta_modules.py.backup.20250730 | 707 ++++++++++++ .../spack-stack/stack/templates/compiler | 14 +- .../spack-stack/stack/templates/compiler.lua | 14 +- .../jcsda-emc/spack-stack/stack/templates/mpi | 7 +- .../spack-stack/stack/templates/mpi.lua | 7 +- .../spack-stack/stack/templates/python | 24 - .../spack-stack/stack/templates/python.lua | 28 - 8 files changed, 1076 insertions(+), 753 deletions(-) create mode 100755 spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250730 delete mode 100644 spack-ext/lib/jcsda-emc/spack-stack/stack/templates/python delete mode 100644 spack-ext/lib/jcsda-emc/spack-stack/stack/templates/python.lua diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 34ba276ff..5388313d6 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -8,6 +8,8 @@ import spack import spack.environment as ev +# import spack.repo +from spack.provider_index import ProviderIndex # logging.basicConfig(level=logging.INFO) logging.basicConfig(format="%(message)s", level=logging.DEBUG) @@ -30,11 +32,10 @@ "lmod": os.path.join(this_script_dir, "templates/mpi.lua"), "tcl": os.path.join(this_script_dir, "templates/mpi"), } -PYTHON_TEMPLATES = { - "lmod": os.path.join(this_script_dir, "templates/python.lua"), - "tcl": os.path.join(this_script_dir, "templates/python"), +MODULE_FILE_EXTENSION = { + "lmod": ".lua", + "tcl": "" } -MODULE_FILE_EXTENSION = {"lmod": ".lua", "tcl": ""} SUBSTITUTES_TEMPLATE = { "MODULELOADS": "", @@ -49,114 +50,16 @@ "MPICC": "", "MPICXX": "", "MPIF77": "", - "MPIFC": "", "MPIROOT": "", - "PYTHONROOT": "", } -def versiontuple(v): - """Version comparison without relying on packaging module""" - return tuple(map(int, (v.split(".")))) - - -def get_name_and_version_from_spec(spec): - """Extract the name and the version from a spec""" - (spec_name, spec_version) = spec.split("@", 1) - # Strip off any compiler/provider specification or variants from the spec version - spec_version = ( - spec_version.split("+")[0].split("~")[0].split("-")[0].split("%")[0].split("^")[0].strip().split(" ")[0].strip() - ) - return (spec_name, spec_version) - - -def get_matched_dict(root_dir, candidate_list, sub_candidate_list=None): - """Return a dictionary of package (compiler, mpi) versions that contains - exact version matches for a list of candidates provided in candidate_list. - This versions are identified by parsing the spack modulefile tree and - matching the candidate names/versions (partially if incomplete) to the - directory names. For packages with dependencies - (e.g. mpi depends on compiler), a recursive search is performed - for compiler versions based on sub_candidate_list.""" - - matched_dict = {} - dirs = [xdir for xdir in os.listdir(root_dir) if os.path.isdir(os.path.join(root_dir, xdir))] - for candidate in candidate_list: - matched_name = None - matched_version = None - for xdir in dirs: - # Partial matches: name/version/ - if ("@" in candidate and xdir == candidate.split("@")[0]) or (xdir == candidate): - candidate_dir = os.path.join(root_dir, xdir) - # Now that we have the top-level compiler dir - # check which versions are installed - versions = [ - ydir - for ydir in os.listdir(candidate_dir) - if os.path.isdir(os.path.join(candidate_dir, ydir)) - ] - # There must be a unique match with the compiler in the candidate list - version_matches = [x for x in versions if candidate in "{}@{}".format(xdir, x)] - # Skip if this entry matches the name but not the version - if not version_matches: - continue - elif len(version_matches) > 1: - raise Exception( - "Multiple version matches for {} in {}: {}".format( - candidate, candidate_dir, version_matches - ) - ) - # Each candidate must match exactly one entry - if matched_name: - raise Exception( - "Multiple matches for {} in {}: {}@{} and {}@{}".format( - candidate, - candidate_dir, - matched_name, - matched_version, - xdir, - version_matches[0], - ) - ) - # Now we have an exact, unique match of mpi name and version - matched_name = xdir - matched_version = version_matches[0] - if sub_candidate_list: - sub_matched_dict = get_matched_dict( - os.path.join(root_dir, matched_name, matched_version), sub_candidate_list - ) - # Partial matches: name-version/ or name-version-hash/ or ... - elif "@" in candidate and xdir.startswith(candidate.replace("@", "-")): - matched_name = candidate.split("@")[0] - matched_version = candidate.split("@")[1] - if not matched_name or not matched_version: - continue - # Compilers for example do not depend on another package - if not sub_candidate_list: - if matched_name not in matched_dict.keys(): - matched_dict[matched_name] = [matched_version] - else: - matched_dict[matched_name].append(matched_version) - # Dedupe - matched_dict[matched_name] = list(set(matched_dict[matched_name])) - # MPI providers or example depend on compilers - else: - if matched_name not in matched_dict.keys(): - matched_dict[matched_name] = {} - if matched_version not in matched_dict[matched_name].keys(): - matched_dict[matched_name][matched_version] = {} - matched_dict[matched_name][matched_version].update(sub_matched_dict) - return matched_dict - - -def merge_dicts(dictA, dictB): - """Merge two dictionaries and remove duplicates""" - for key in dictB.keys(): - if not key in dictA.keys(): - dictA[key] = copy.deepcopy(dictB[key]) - else: - dictA[key] = list(set(dictA[key] + dictB[key])) - return dictA +COMPILER_TRANSLATION_TABLE = { + "gcc" : "gcc", + "clang" : "llvm", + "intel" : "intel-oneapi-compilers-classic", + "oneapi" : "intel-oneapi-compilers", +} def setenv_command(module_choice, key, value): @@ -166,6 +69,13 @@ def setenv_command(module_choice, key, value): return "setenv {{{}}} {{{}}}\n".format(key, value) +def unsetenv_command(module_choice, key, value): + if module_choice == "lmod": + return 'unsetenv("{}", "{}")\n'.format(key, value) + else: + return "unsetenv {{{}}} {{{}}}\n".format(key, value) + + def append_path_command(module_choice, key, value): if module_choice == "lmod": return 'append_path("{}", "{}")\n'.format(key, value) @@ -260,11 +170,6 @@ def setup_meta_modules(): install_dir = os.path.realpath(install_dir) logging.info(" ... install directory: {}".format(install_dir)) - ################################################################## - # Parse the modulefile directory to determine the combinations # - # of compiler and mpi providers for which to create meta modules # - ################################################################## - # Parse spack module config from environment logging.info("Parsing spack environment modules config ...") module_config = spack.config.get("modules") @@ -290,55 +195,31 @@ def setup_meta_modules(): module_dir = os.path.realpath(os.path.join(env_dir, module_dir)) else: module_dir = os.path.realpath(module_dir) - logging.info(" ... module directory: {}".format(module_dir)) - - # Parse spack package config from environment - logging.info("Parsing spack environment package config ...") - package_config = spack.config.get("packages") - compiler_candidate_list = package_config["all"]["compiler"] - logging.debug(" ... list of possible compilers: '{}'".format(compiler_candidate_list)) - mpi_candidate_list = package_config["all"]["providers"]["mpi"] - logging.debug(" ... list of possible mpi providers: '{}'".format(mpi_candidate_list)) - - # Sanity checks: compiler_candidate_list and mpi_candidate_list must be unique - if not len(compiler_candidate_list) == len(set(compiler_candidate_list)): - raise Exception( - "Compiler candidate list is not unique: {}".format(compiler_candidate_list) - ) - if not len(mpi_candidate_list) == len(set(mpi_candidate_list)): - raise Exception("MPI candidate list is not unique: {}".format(mpi_candidate_list)) - - # Parse the directory tree under the top-level module directory - logging.debug(os.listdir(module_dir)) - # First, check for compilers - compiler_dict = get_matched_dict(module_dir, compiler_candidate_list) - if not compiler_dict: - raise Exception("No matching compilers found") - logging.info(" ... stack compilers: '{}'".format(compiler_dict)) - - # Then, check for mpi providers - recursively for compilers - mpi_dict = get_matched_dict(module_dir, mpi_candidate_list, compiler_candidate_list) - if not mpi_dict: - logging.warn("No matching MPI providers found, skipping MPI module hierarchy") - else: - logging.info(" ... stack mpi providers: '{}'".format(mpi_dict)) - - # For some environments, there are only compiler+mpi-dependent modules, - # and therefore the compiler itself is not recorded in compiler_dict. - for mpi_provider_name in mpi_dict.keys(): - for mpi_provider_version in mpi_dict[mpi_provider_name].keys(): - compiler_dict_tmp = get_matched_dict( - os.path.join(module_dir, mpi_provider_name, mpi_provider_version), - compiler_candidate_list, - ) - compiler_dict = merge_dicts(compiler_dict, compiler_dict_tmp) + logging.info(f" ... module directory: {module_dir}") + + # Get all specs and determine compilers and mpi providers + hashes = env.all_hashes() + specs = spack.store.STORE.db.query(hashes=hashes) + q = ProviderIndex(specs=specs, repository=spack.repo.PATH) + + c_providers = q.providers_for("c") + cxx_providers = q.providers_for("cxx") + fortran_providers = q.providers_for("fortran") + compilers = list(set(c_providers + cxx_providers + fortran_providers)) + if not compilers: + raise Exception("No compilers found") + logging.info(f" ... compilers: {compilers}") + + # To remove compiler prefices from tcl modulefiles, we need + # a mock compiler "none@none" for external packages + if module_choice == "tcl": + mock_compiler_list = [x.name+"@"+str(x.version) for x in compilers] + ["none@none"] + logging.debug(f" ... mock_compiler_list: {mock_compiler_list}") - # For future use, we need a flattened list of all compilers - flattened_compiler_list = [ - "{}@{}".format(name, version) - for name in compiler_dict.keys() - for version in compiler_dict[name] - ] + mpi_providers = q.providers_for("mpi") + if not len(mpi_providers)==1: + raise Exception(f"Expected exactly one MPI provider, but got {mpi_providers}") + logging.info(f" ... mpi_providers: {mpi_providers}") # Core compilers is only a valid options for lmod if module_choice == "lmod": @@ -360,64 +241,250 @@ def setup_meta_modules(): os.mkdir(meta_module_dir) logging.info(" ... meta module directory : {}".format(meta_module_dir)) - # Determine the preferred compiler and sort the flattened list of compilers - # such that the preferred compiler comes last. This is so that all other - # compilers populate the MODULEPATHS_SAVE list before the preferred compiler + + # DH* 20250730 - DO WE NEED BOTH WAYS TO DETERMINE THE PREFERRED COMPILER? + # MAYBE KEEP FOR NOW UNTIL WE KNOW BETTER WHAT IS NEEDED AND WHAT NOT. + + # Determine the preferred compiler and sort the list of compilers so that + # the preferred compiler comes last. This is so that all other compilers + # populate the MODULEPATHS_SAVE list before the preferred compiler # takes it and adds it to the stack-COMPILER metamodule. Likewise, we need # to save the list of compiler substitutions from the preferred compiler - # so that we have access to it when we build the MPI meta module. Note that - # by definition, only the preferred compiler can be used for MPI dependencies. + # so that we have access to it when we build the MPI meta module. try: - preferred_compilers = spack.config.get("packages")["all"]["prefer"] + preferred_compilers = spack.config.get("packages")["all"]["require"] except: raise Exception( """Unable to detect preferred compiler from environment. - Does the environment have the config entry 'packages:all:prefer?'""" + Does the environment have the config entry 'packages:all:require?'""" ) if len(preferred_compilers)>1: - raise Exception(f"Not supported: more than one preferred compiler: {preferred_compilers}") - preferred_compiler = preferred_compilers[0].replace("%","") + raise Exception(f"Invalid value for packages:all:require is {preferred_compilers}") + match = re.search(r'%\[when=%fortran\](\S+)', preferred_compilers[0]) + if match and match.group(1) in COMPILER_TRANSLATION_TABLE.keys(): + preferred_compiler_v1 = COMPILER_TRANSLATION_TABLE[match.group(1)] + else: + raise Exception(f"Invalid value for packages:all:require {preferred_compilers[0]}") + + try: + preferred_compilers = spack.config.get("packages")["fortran"]["require"] + except: + raise Exception( + """Unable to detect preferred compiler from environment. + Does the environment have the config entry 'packages:fortran:require?'""" + ) + if len(preferred_compilers)>1: + raise Exception(f"Invalid value for packages:fortran:require is {preferred_compilers}") + if preferred_compilers[0] in COMPILER_TRANSLATION_TABLE.values(): + preferred_compiler_v2 = preferred_compilers[0] + else: + raise Exception(f"Invalid value for packages:fortran:require {preferred_compilers[0]}") + if preferred_compiler_v1 == preferred_compiler_v2: + preferred_compiler = preferred_compiler_v1 + del preferred_compilers + del preferred_compiler_v1 + del preferred_compiler_v2 + else: + raise Exception(f"Multiple preferred compilers in spack config: {preferred_compiler_v1} vs {preferred_compiler_v2}") logging.info(" ... preferred compiler: {}".format(preferred_compiler)) - del preferred_compilers + # *DH 20250730 + # Sort the list using a custom key def custom_sort_key(entry): # Return a tuple where the first element is 1 if the entry contains the word, else 0 # The second element is the entry itself for natural sorting within groups return (1 if preferred_compiler in entry else 0, entry) - sorted_flattened_compiler_list = sorted(flattened_compiler_list, key=custom_sort_key) + compilers = sorted(compilers, key=custom_sort_key) # Create compiler modules logging.info("Creating compiler modules ...") - compiler_config = spack.config.get("compilers") + # Collect and save modulepaths for the preferred compiler MODULEPATHS_SAVE = [] + + # Append modulepath for external specs + modulepath_save = os.path.join(module_dir, "none", "none") + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + logging.info(" ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) + MODULEPATHS_SAVE.append(modulepath_save) + # Initialize saved substitutes to None (populate for preferred compiler later) - SUBSTITUTES_SAVE = None - for compiler_identifier in sorted_flattened_compiler_list: - (compiler_name, compiler_version) = compiler_identifier.replace("@=","@").split("@") - # Loop through all configured compilers and find the correct match - compiler = False - for candidate_compiler in compiler_config: - if compiler_identifier.replace("@=","@") == \ - candidate_compiler["compiler"]["spec"].replace("@=", "@"): - compiler = candidate_compiler - break - if not compiler: - raise Exception("No matching compiler for {} found in spack compiler config".format(compiler)) + COMPILER_SUBSTITUTES_SAVE = None + + for compiler in compilers: + logging.info(f" ... configuring compiler {compiler.name}@{compiler.version}") + + modulepath_save = os.path.join(module_dir, compiler.name, str(compiler.version)) + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + logging.info(" ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) + MODULEPATHS_SAVE.append(modulepath_save) + + # For tcl modules remove the compiler prefices from the module contents + if module_choice == "tcl": + logging.info( + " ... ... removing compiler prefices from tcl modulefiles in {}".format( + modulepath_save + ) + ) + + for root, ddir, files in os.walk(modulepath_save): + for ffile in files: + filepath = os.path.join(root, ffile) + logging.debug( + " ... ... ... removing compiler prefices in {}".format(filepath) + ) + for pattern in module_replace_patterns: + for tmp_compiler in mock_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") + cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( + pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + + # The remainder of the loop is only needed for the preferred compiler + if not compiler.name in preferred_compiler: continue - # This is only to keep the current level of indentation for easier reviewing - if True: - modulepath_save = os.path.join(module_dir, compiler_name, compiler_version) + logging.info( + " ... configuring stack compiler {}@{}".format(compiler.name, compiler.version) + ) + compiler_module_dir = os.path.join(meta_module_dir, "stack-" + compiler.name) + compiler_module_file = os.path.join( + compiler_module_dir, str(compiler.version) + MODULE_FILE_EXTENSION[module_choice] + ) + substitutes = SUBSTITUTES_TEMPLATE.copy() + + if not compiler.external: + raise Exception(f"spack-built compiler {compiler} not yet supported") + + # Use existing modules for external mpi providers; otherwise, use spack-built module + if compiler.external and compiler.external_modules: + raise Exception(f"UNTESTED XYZ MODULES: {compiler.external_modules}") + for module in compiler.external_modules: + substitutes["MODULELOADS"] += module_load_command(module_choice, module) + substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) + else: + module = "{}/{}".format(compiler.name, compiler.version) + substitutes["MODULELOADS"] += module_load_command(module_choice, module) + substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) + substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") + substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") + logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) + logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) + + # Compiler environment variables; names are lowercase in spack + substitutes["CC"] = compiler.extra_attributes["compilers"]["c"] + substitutes["CXX"] = compiler.extra_attributes["compilers"]["cxx"] + substitutes["F77"] = compiler.extra_attributes["compilers"]["fortran"] + substitutes["FC"] = compiler.extra_attributes["compilers"]["fortran"] + logging.debug(" ... ... CC : {}".format(substitutes["CC"])) + logging.debug(" ... ... CXX : {}".format(substitutes["CXX"])) + logging.debug(" ... ... F77 : {}".format(substitutes["F77"])) + logging.debug(" ... ... FC' : {}".format(substitutes["FC"])) + + # Compiler flags; names are lowercase in spack + if "flags" in compiler.extra_attributes.keys(): + for flag_name in compiler.extra_attributes["flags"].keys(): + flag_values = compiler["compiler"]["flags"][flag_name] + substitutes["COMPFLAGS"] += setenv_command( + module_choice, flag_name.upper(), flag_values + ) + substitutes["COMPFLAGS"] = substitutes["COMPFLAGS"].rstrip("\n") + logging.debug(" ... ... COMPFLAGS: {}".format(substitutes["COMPFLAGS"])) + + # Environment variables; case-sensitive in spack + if "environment" in compiler.extra_attributes.keys(): + # DH* CAN WE SIMPLIFY THIS WITH A LOOP AND A GENERIC FUNCTION? + # append_path + if "append_path" in compiler.extra_attributes["environment"].keys(): + for env_name in compiler.extra_attributes["environment"]["append_path"]: + env_values = compiler.extra_attributes["environment"]["append_path"][env_name] + substitutes["ENVVARS"] += append_path_command( + module_choice, env_name, env_values + ) + # prepend_path + if "prepend_path" in compiler.extra_attributes["environment"].keys(): + for env_name in compiler.extra_attributes["environment"]["prepend_path"]: + env_values = compiler.extra_attributes["environment"]["prepend_path"][env_name] + substitutes["ENVVARS"] += prepend_path_command( + module_choice, env_name, env_values + ) + # set + if "set" in compiler.extra_attributes["environment"].keys(): + for env_name in compiler.extra_attributes["environment"]["set"]: + env_values = compiler.extra_attributes["environment"]["set"][env_name] + substitutes["ENVVARS"] += setenv_command( + module_choice, env_name, env_values + ) + # unset + if "unset" in compiler.extra_attributes["environment"].keys(): + for env_name in compiler.extra_attributes["environment"]["unset"]: + env_values = compiler.extra_attributes["environment"]["unset"][env_name] + substitutes["ENVVARS"] += unsetenv_command( + module_choice, env_name, env_values + ) + # *DH + substitutes["ENVVARS"] = substitutes["ENVVARS"].rstrip("\n") + logging.debug(" ... ... ENVVARS : {}".format(substitutes["ENVVARS"])) + + # Spack compiler module hierarchy - append all saved modulepaths + for modulepath in MODULEPATHS_SAVE: + substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) + substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") + logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) + + # Read compiler template into module_content string + with open(COMPILER_TEMPLATES[module_choice]) as f: + module_content = f.read() + + # Substitute variables in module_content + for key in substitutes.keys(): + module_content = module_content.replace("@{}@".format(key), substitutes[key]) + + # Write compiler lua module + if not os.path.isdir(compiler_module_dir): + os.makedirs(compiler_module_dir) + with open(compiler_module_file, "w") as f: + f.write(module_content) + logging.info(" ... writing {}".format(compiler_module_file)) + # If this is the last compiler in the list (i.e. the preferred compiler), + # then save the substitutes for later use for building the MPI meta module. + if compiler == compilers[-1]: + COMPILER_SUBSTITUTES_SAVE = substitutes + + del MODULEPATHS_SAVE + + # Collect and save modulepaths for MPI with preferred compiler + MODULEPATHS_SAVE = [] + + # Create mpi modules - as of July 2025, only one + # MPI provider is allowed (for the preferred compiler) + for mpi_provider in mpi_providers: + + for compiler in compilers: + logging.info( + " ... configuring stack mpi library {}@{} for compiler {}@{}".format( + mpi_provider.name, mpi_provider.version, compiler.name, compiler.version + ) + ) + + # Spack mpi+compiler module hierarchy + modulepath_save = os.path.join( + module_dir, mpi_provider.name, str(mpi_provider.version), compiler.name, str(compiler.version) + ) if not os.path.isdir(modulepath_save): os.makedirs(modulepath_save) logging.info(" ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) MODULEPATHS_SAVE.append(modulepath_save) - # For tcl modules remove the compiler prefices from the module contents + # For tcl modules remove the compiler/mpi prefices from the module contents if module_choice == "tcl": logging.info( - " ... ... removing compiler prefices from tcl modulefiles in {}".format( + " ... ... removing compiler/mpi prefices from tcl modulefiles in {}".format( modulepath_save ) ) @@ -425,55 +492,62 @@ def custom_sort_key(entry): for ffile in files: filepath = os.path.join(root, ffile) logging.debug( - " ... ... ... removing compiler prefices in {}".format(filepath) + " ... ... ... removing compiler/mpi prefices in {}".format( + filepath + ) ) + # First, compiler-only dependent modules + # These can depend on other compilers than the + # compiler this MPI was built with, loop over all for pattern in module_replace_patterns: - for tmp_compiler_spec in sorted_flattened_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler_spec.split("@") + for tmp_compiler in mock_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix ) status = os.system(cmd) if not status == 0: raise Exception("Error while calling '{}'".format(cmd)) - # Paranoid, but better safe than sorry - del pattern - del tmp_compiler_spec + # Then, compiler+mpi-dependent modules + # By definition, these can only depend on the compiler that + # this MPI was built with - no need to loop over all compilers + for pattern in module_replace_patterns: + for tmp_compiler in mock_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") + cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format( + pattern, + mpi_provider.name, + mpi_provider.version, + tmp_compiler_name, + tmp_compiler_version, + filepath, + sed_syntax_fix, + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) # The remainder of the loop is only needed for the preferred compiler - if not compiler_name in preferred_compiler: + if not compiler.name in preferred_compiler: continue - logging.info( - " ... configuring stack compiler {}@{}".format(compiler_name, compiler_version) + # Path and name for mpi module file + mpi_module_dir = os.path.join( + module_dir, compiler.name, str(compiler.version), "stack-" + mpi_provider.name ) - compiler_module_dir = os.path.join(meta_module_dir, "stack-" + compiler_name) - compiler_module_file = os.path.join( - compiler_module_dir, compiler_version + MODULE_FILE_EXTENSION[module_choice] + mpi_module_file = os.path.join( + mpi_module_dir, str(mpi_provider.version).split("-")[0] + MODULE_FILE_EXTENSION[module_choice] ) - substitutes = SUBSTITUTES_TEMPLATE.copy() - # Compiler environment variables; names are lowercase in spack - substitutes["CC"] = compiler["compiler"]["paths"]["cc"] - substitutes["CXX"] = compiler["compiler"]["paths"]["cxx"] - substitutes["F77"] = compiler["compiler"]["paths"]["f77"] - substitutes["FC"] = compiler["compiler"]["paths"]["fc"] - logging.debug(" ... ... CC : {}".format(substitutes["CC"])) - logging.debug(" ... ... CXX : {}".format(substitutes["CXX"])) - logging.debug(" ... ... F77 : {}".format(substitutes["F77"])) - logging.debug(" ... ... FC' : {}".format(substitutes["FC"])) - - # Compiler flags; names are lowercase in spack - for flag_name in compiler["compiler"]["flags"]: - flag_values = compiler["compiler"]["flags"][flag_name] - substitutes["COMPFLAGS"] += setenv_command( - module_choice, flag_name.upper(), flag_values - ) - substitutes["COMPFLAGS"] = substitutes["COMPFLAGS"].rstrip("\n") - logging.debug(" ... ... COMPFLAGS: {}".format(substitutes["COMPFLAGS"])) - - # Existing non-spack modules to load - for module in compiler["compiler"]["modules"]: + substitutes = SUBSTITUTES_TEMPLATE.copy() + # Use existing modules for external mpi providers; otherwise, use spack-built module + if mpi_provider.external and mpi_provider.external_modules: + raise Exception(f"UNTESTED XYZ MPI MODULES: {mpi_provider.external_modules}") + for module in mpi_provider.external_modules: + substitutes["MODULELOADS"] += module_load_command(module_choice, module) + substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) + else: + module = "{}/{}".format(mpi_provider.name, mpi_provider.version) substitutes["MODULELOADS"] += module_load_command(module_choice, module) substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") @@ -481,482 +555,58 @@ def custom_sort_key(entry): logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) - # Environment variables; case-sensitive in spack - if ( - "environment" in compiler["compiler"].keys() - and compiler["compiler"]["environment"] - ): - # append_path - if "append_path" in compiler["compiler"]["environment"].keys(): - for env_name in compiler["compiler"]["environment"]["append_path"]: - env_values = compiler["compiler"]["environment"]["append_path"][env_name] - substitutes["ENVVARS"] += append_path_command( - module_choice, env_name, env_values - ) - # prepend_path - if "prepend_path" in compiler["compiler"]["environment"].keys(): - for env_name in compiler["compiler"]["environment"]["prepend_path"]: - env_values = compiler["compiler"]["environment"]["prepend_path"][env_name] - substitutes["ENVVARS"] += prepend_path_command( - module_choice, env_name, env_values - ) - # set - if "set" in compiler["compiler"]["environment"].keys(): - for env_name in compiler["compiler"]["environment"]["set"]: - env_values = compiler["compiler"]["environment"]["set"][env_name] - substitutes["ENVVARS"] += setenv_command( - module_choice, env_name, env_values - ) - substitutes["ENVVARS"] = substitutes["ENVVARS"].rstrip("\n") - logging.debug(" ... ... ENVVARS : {}".format(substitutes["ENVVARS"])) - - # Spack compiler module hierarchy - append all saved modulepaths + # Compiler wrapper environment variables + if mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers": + substitutes["MPICC"] = os.path.join("mpiicx") + substitutes["MPICXX"] = os.path.join("mpiicpx") + if "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and not "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: + substitutes["MPIF77"] = os.path.join("mpiifx") + substitutes["MPIF90"] = os.path.join("mpiifx") + elif not "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: + substitutes["MPIF77"] = os.path.join("mpiifort") + substitutes["MPIF90"] = os.path.join("mpiifort") + else: + raise Exception(f"For {mpi_provider.name}, cannot determine MPI wrapper from FC={COMPILER_SUBSTITUTES_SAVE['FC']}") + elif mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers-classic": + substitutes["MPICC"] = os.path.join("mpiicc") + substitutes["MPICXX"] = os.path.join("mpiicpc") + substitutes["MPIF77"] = os.path.join("mpiifort") + substitutes["MPIF90"] = os.path.join("mpiifort") + else: + substitutes["MPICC"] = os.path.join("mpicc") + substitutes["MPICXX"] = os.path.join("mpic++") + substitutes["MPIF77"] = os.path.join("mpif77") + substitutes["MPIF90"] = os.path.join("mpif90") + # Also set the direct compiler environment variables + substitutes["CC"] = COMPILER_SUBSTITUTES_SAVE["CC"] + substitutes["CXX"] = COMPILER_SUBSTITUTES_SAVE["CXX"] + substitutes["F77"] = COMPILER_SUBSTITUTES_SAVE["F77"] + substitutes["FC"] = COMPILER_SUBSTITUTES_SAVE["FC"] + + # Spack mpi+compiler module hierarchy - append all saved modulepaths for modulepath in MODULEPATHS_SAVE: substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) - # Read compiler template into module_content string - with open(COMPILER_TEMPLATES[module_choice]) as f: - module_content = f.read() - - # Substitute variables in module_content - for key in substitutes.keys(): - module_content = module_content.replace("@{}@".format(key), substitutes[key]) - - # Write compiler lua module - if not os.path.isdir(compiler_module_dir): - os.makedirs(compiler_module_dir) - with open(compiler_module_file, "w") as f: - f.write(module_content) - logging.info(" ... writing {}".format(compiler_module_file)) - # If this is the last compiler in the list (i.e. the preferred compiler), - # then save the substitutes for later use for building the MPI meta module. - if compiler_identifier == sorted_flattened_compiler_list[-1]: - SUBSTITUTES_SAVE = substitutes - del MODULEPATHS_SAVE - - # Create mpi modules - for mpi_name in mpi_dict.keys(): - for mpi_version in mpi_dict[mpi_name].keys(): - package_found = False - external_mpi_package_config = None - for package_name in package_config.keys(): - if package_name == mpi_name: - # mpi provider was supplied as an external package - if "externals" in package_config[package_name].keys(): - for i in range(len(package_config[package_name]["externals"])): - (package_name_dummy, package_version) = get_name_and_version_from_spec( - package_config[package_name]["externals"][i]["spec"] - ) - if package_version == mpi_version.split("-")[0]: - package_found = True - external_mpi_package_config = package_config[package_name][ - "externals" - ][i] - break - # mpi provider was built by spack - elif ( - "version" in package_config[package_name].keys() - and len(package_config[package_name]["version"]) == 1 - and package_config[package_name]["version"][0] == mpi_version.split("-")[0] - ): - package_found = True - break - # mpi provider was built by spack, we don't have enough - # information, just hope we will be lucky - elif "version" not in package_config[package_name].keys(): - package_found = True - break - else: - raise Exception( - "Package with matching name is incompatible: {}".format( - package_config[package_name] - ) - ) - if package_found: - break - if not package_found: - raise Exception( - "Could not find a spack package for {}@{}:".format(mpi_name, mpi_version) - ) - - for compiler_name in mpi_dict[mpi_name][mpi_version].keys(): - for compiler_version in mpi_dict[mpi_name][mpi_version][compiler_name]: - logging.info( - " ... configuring stack mpi library {}@{} for compiler {}@{}".format( - mpi_name, mpi_version, compiler_name, compiler_version - ) - ) - - # Path and name for lua module file - mpi_module_dir = os.path.join( - module_dir, compiler_name, compiler_version, "stack-" + mpi_name - ) - mpi_module_file = os.path.join( - mpi_module_dir, mpi_version.split("-")[0] + MODULE_FILE_EXTENSION[module_choice] - ) - substitutes = SUBSTITUTES_TEMPLATE.copy() - # - if ( - external_mpi_package_config - and "modules" in external_mpi_package_config.keys() - ): - # Existing non-spack modules to load - for module in external_mpi_package_config["modules"]: - substitutes["MODULELOADS"] += module_load_command( - module_choice, module - ) - substitutes["MODULEPREREQS"] += module_prereq_command( - module_choice, module - ) - substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") - substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") - logging.debug( - " ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"]) - ) - logging.debug( - " ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"]) - ) - # mpi_name_ROOT - replace "-" in mpi_name with "_" for environment variables - if "prefix" in external_mpi_package_config.keys(): - prefix = external_mpi_package_config["prefix"] - substitutes["MPIROOT"] = setenv_command( - module_choice, mpi_name.replace("-", "_") + "_ROOT", prefix - ) - logging.debug( - " ... ... MPIROOT: {}".format(substitutes["MPIROOT"]).rstrip("\n") - ) - elif ( - external_mpi_package_config - and "prefix" in external_mpi_package_config.keys() - ): - prefix = external_mpi_package_config["prefix"] - # PATH and compiler wrapper environment variables - bindir = os.path.join(prefix, "bin") - if os.path.isdir(bindir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "PATH", bindir - ) - # LD_LIBRARY_PATH AND PKG_CONFIG_PATH - do we need to worry about DYLD_LIBRARY_PATH for macOS? - libdir = os.path.join(prefix, "lib") - if os.path.isdir(libdir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "LD_LIBRARY_PATH", libdir - ) - pkgconfigdir = os.path.join(libdir, "pkgconfig") - if os.path.isdir(pkgconfigdir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "PKG_CONFIG_PATH", pkgconfigdir - ) - lib64dir = os.path.join(prefix, "lib64") - if os.path.isdir(lib64dir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "LD_LIBRARY_PATH", lib64dir - ) - pkgconfig64dir = os.path.join(lib64dir, "pkgconfig") - if os.path.isdir(pkgconfig64dir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "PKG_CONFIG_PATH", pkgconfig64dir - ) - # MANPATH - mandir = os.path.join(prefix, "share/man") - if os.path.isdir(mandir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "MANPATH", mandir - ) - # ACLOCAL_PATH - aclocaldir = os.path.join(prefix, "share/aclocal") - if os.path.isdir(aclocaldir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "ACLOCAL_PATH", aclocaldir - ) - # mpi_name_ROOT - replace "-" in mpi_name with "_" for environment variables - substitutes["MPIROOT"] = setenv_command( - module_choice, mpi_name.replace("-", "_") + "_ROOT", prefix - ) - logging.debug( - " ... ... MPIROOT: {}".format(substitutes["MPIROOT"]).rstrip("\n") - ) - else: - # Simple workaround for now - depend on spack building the mpi module - # and load it with the wrapper, don't set any of the other substitutes - module = "{}/{}".format(mpi_name, mpi_version) - substitutes["MODULELOADS"] += module_load_command( - module_choice, module.split("-")[0] - ).rstrip("\n") - substitutes["MODULEPREREQS"] += module_prereq_command( - module_choice, module.split("-")[0] - ).rstrip("\n") - logging.debug( - " ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"]) - ) - logging.debug( - " ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"]) - ) - - # Compiler wrapper environment variables - if "intel" in mpi_name and compiler_name == "oneapi": - substitutes["MPICC"] = os.path.join("mpiicx") - substitutes["MPICXX"] = os.path.join("mpiicpx") - if "ifx" in SUBSTITUTES_SAVE["FC"] and not "ifort" in SUBSTITUTES_SAVE["FC"]: - substitutes["MPIF77"] = os.path.join("mpiifx") - substitutes["MPIF90"] = os.path.join("mpiifx") - elif not "ifx" in SUBSTITUTES_SAVE["FC"] and "ifort" in SUBSTITUTES_SAVE["FC"]: - substitutes["MPIF77"] = os.path.join("mpiifort") - substitutes["MPIF90"] = os.path.join("mpiifort") - else: - raise Exception(f"For {mpi_name}, cannot determine MPI wrapper from FC={SUBSTITUTES_SAVE['FC']}") - elif "intel" in mpi_name and compiler_name == "intel": - substitutes["MPICC"] = os.path.join("mpiicc") - substitutes["MPICXX"] = os.path.join("mpiicpc") - substitutes["MPIF77"] = os.path.join("mpiifort") - substitutes["MPIF90"] = os.path.join("mpiifort") - else: - substitutes["MPICC"] = os.path.join("mpicc") - substitutes["MPICXX"] = os.path.join("mpic++") - substitutes["MPIF77"] = os.path.join("mpif77") - substitutes["MPIF90"] = os.path.join("mpif90") - # Also set the direct compiler environment variables - substitutes["CC"] = SUBSTITUTES_SAVE["CC"] - substitutes["CXX"] = SUBSTITUTES_SAVE["CXX"] - substitutes["F77"] = SUBSTITUTES_SAVE["F77"] - substitutes["FC"] = SUBSTITUTES_SAVE["FC"] - - # Spack mpi+compiler module hierarchy - modulepath = os.path.join( - module_dir, mpi_name, mpi_version, compiler_name, compiler_version - ) - # If the environment doesn't have mpi dependent modules - # then simply create the placeholder directory - if not os.path.isdir(modulepath): - os.makedirs(modulepath) - substitutes["MODULEPATHS"] = modulepath_prepend_command(module_choice, modulepath).rstrip("\n") - logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) - - # For tcl modules remove the compiler/mpi prefices from the module contents - if module_choice == "tcl": - logging.info( - " ... ... removing compiler/mpi prefices from tcl modulefiles in {}".format( - modulepath - ) - ) - for root, ddir, files in os.walk(modulepath): - for ffile in files: - filepath = os.path.join(root, ffile) - logging.debug( - " ... ... ... removing compiler/mpi prefices in {}".format( - filepath - ) - ) - # Search patterns - patterns = ["is-loaded", "module load", "depends-on"] - # First, compiler-only dependent modules - # These can depend on other compilers than the - # compiler this MPI was built with, loop over all - for pattern in module_replace_patterns: - for tmp_compiler_spec in sorted_flattened_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler_spec.split("@") - cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( - pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) - # Paranoid, but better safe than sorry - del pattern - del tmp_compiler_spec - # Then, compiler+mpi-dependent modules - # By definition, these can only depend on the compiler that - # this MPI was built with - no need to loop over all compilers - for pattern in module_replace_patterns: - cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format( - pattern, - mpi_name, - mpi_version, - compiler_name, - compiler_version, - filepath, - sed_syntax_fix, - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) - - # Read compiler lua template into module_content string - with open(MPI_TEMPLATES[module_choice]) as f: - module_content = f.read() - - # Substitute variables in module_content - for key in substitutes.keys(): - module_content = module_content.replace( - "@{}@".format(key), substitutes[key] - ) - - # Write mpi lua module - if not os.path.isdir(mpi_module_dir): - os.makedirs(mpi_module_dir) - with open(mpi_module_file, "w") as f: - f.write(module_content) - logging.info(" ... writing {}".format(mpi_module_file)) - del SUBSTITUTES_SAVE - - # Create python modules. Need to accommodate both external - # Python distributions and spack-built Python distributions. - python_dict = {} - python_name = "python" - - # First, collect all Python packages in the environment - for spec in ev.installed_specs(): - if spec.name == python_name: - compiler_name = spec.compiler.name - compiler_version = str(spec.compiler.version) - if not compiler_name in python_dict.keys(): - python_dict[compiler_name] = {} - if compiler_version in python_dict[compiler_name].keys(): - raise Exception("Duplicate Python packages for compiler {spec.compiler}") - python_dict[compiler_name][compiler_version] = spec - - for compiler_name in compiler_dict.keys(): - for compiler_version in compiler_dict[compiler_name]: - if ( - not compiler_name in python_dict.keys() - or not compiler_version in python_dict[compiler_name].keys() - ): - continue - spec = python_dict[compiler_name][compiler_version] - python_version = str(spec.version) - logging.info( - f" ... configuring stack python interpreter {python_name}@{python_version} for compiler {compiler_name}@{compiler_version}" - ) - - if spec.external: - logging.debug(f" ... using external python version {python_version}") - prefix = spec.external_path - modules = spec.external_modules - else: - logging.debug(f" ... using spack-built python version {python_version}") - - # Path and name for lua module file - python_module_dir = os.path.join( - module_dir, compiler_name, compiler_version, "stack-" + python_name - ) - python_module_file = os.path.join( - python_module_dir, python_version + MODULE_FILE_EXTENSION[module_choice] - ) - - substitutes = SUBSTITUTES_TEMPLATE.copy() - # Spack-built Python vs external Python - if not spec.external: - module = "python/{}".format(python_version) - # Load spack python module - substitutes["MODULELOADS"] += module_load_command(module_choice, module) - elif modules: - # Existing non-spack modules to load - for module in python_package_config["modules"]: - substitutes["MODULELOADS"] += module_load_command(module_choice, module) - substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) - substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") - substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") - logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) - logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) - # python_name_ROOT - replace "-" in python_name with "_" for environment variables - if prefix: - substitutes["PYTHONROOT"] = setenv_command( - module_choice, python_name.replace("-", "_") + "_ROOT", prefix - ) - logging.debug(" ... ... PYTHONROOT: {}".format(substitutes["PYTHONROOT"])) - elif prefix: - # PATH - bindir = os.path.join(prefix, "bin") - if os.path.isdir(bindir): - substitutes["ENVVARS"] += prepend_path_command(module_choice, "PATH", bindir) - # LD_LIBRARY_PATH AND PKG_CONFIG_PATH - do we need to worry about DYLD_LIBRARY_PATH for macOS? - # Also: PYTHONPATH = check site-packages and dist-packages - libdir = os.path.join(prefix, "lib") - if os.path.isdir(libdir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "LD_LIBRARY_PATH", libdir - ) - pkgconfigdir = os.path.join(libdir, "pkgconfig") - if os.path.isdir(pkgconfigdir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "PKG_CONFIG_PATH", pkgconfigdir - ) - # Python version for constructing PYTHONPATH is X.Y (major.minor, no patch-level) - python_version_for_pythonpath = python_version[: python_version.rfind(".")] - # Check site-packages and dist-packages - pythonpathdir = os.path.join( - libdir, "python{}".format(python_version_for_pythonpath), "site-packages" - ) - if os.path.isdir(pythonpathdir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "PYTHONPATH", pythonpathdir - ) - pythonpathdir = os.path.join( - libdir, "python{}".format(python_version_for_pythonpath), "dist-packages" - ) - if os.path.isdir(pythonpathdir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "PYTHONPATH", pythonpathdir - ) - lib64dir = os.path.join(prefix, "lib64") - if os.path.isdir(lib64dir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "LD_LIBRARY_PATH", lib64dir - ) - pkgconfig64dir = os.path.join(lib64dir, "pkgconfig") - if os.path.isdir(pkgconfig64dir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "PKG_CONFIG_PATH", pkgconfig64dir - ) - pythonpath64dir = os.path.join( - lib64dir, "python{}".format(python_version_for_pythonpath), "site-packages" - ) - if os.path.isdir(pythonpath64dir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "PYTHONPATH", pythonpath64dir - ) - pythonpath64dir = os.path.join( - lib64dir, "python{}".format(python_version_for_pythonpath), "dist-packages" - ) - if os.path.isdir(pythonpath64dir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "PYTHONPATH", pythonpath64dir - ) - # MANPATH - mandir = os.path.join(prefix, "share/man") - if os.path.isdir(mandir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "MANPATH", mandir - ) - # ACLOCAL_PATH - aclocaldir = os.path.join(prefix, "share/aclocal") - if os.path.isdir(aclocaldir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "ACLOCAL_PATH", aclocaldir - ) - # python_name_ROOT - replace "-" in - # python_name with "_" for environment variables - substitutes["PYTHONROOT"] = setenv_command( - module_choice, python_name.replace("-", "_") + "_ROOT", prefix - ) - else: - raise Exception("External packages must have 'prefix' and/or 'modules'") - # Read compiler lua template into module_content string - with open(PYTHON_TEMPLATES[module_choice]) as f: + with open(MPI_TEMPLATES[module_choice]) as f: module_content = f.read() # Substitute variables in module_content for key in substitutes.keys(): - module_content = module_content.replace("@{}@".format(key), substitutes[key]) + module_content = module_content.replace( + "@{}@".format(key), substitutes[key] + ) - # Write python lua module - if not os.path.isdir(python_module_dir): - os.makedirs(python_module_dir) - with open(python_module_file, "w") as f: + # Write mpi lua module + if not os.path.isdir(mpi_module_dir): + os.makedirs(mpi_module_dir) + with open(mpi_module_file, "w") as f: f.write(module_content) - logging.info(" ... writing {}".format(python_module_file)) + logging.info(" ... writing {}".format(mpi_module_file)) + + del COMPILER_SUBSTITUTES_SAVE + del MODULEPATHS_SAVE logging.info("Metamodule generation completed successfully in {}".format(meta_module_dir)) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250730 b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250730 new file mode 100755 index 000000000..25dd2446f --- /dev/null +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250730 @@ -0,0 +1,707 @@ +#!/usr/bin/env python3 + +import copy +import logging +import os +import re +import sys + +import spack +import spack.environment as ev +# import spack.repo +from spack.provider_index import ProviderIndex + +# logging.basicConfig(level=logging.INFO) +logging.basicConfig(format="%(message)s", level=logging.DEBUG) + +# Get basic directory information +logging.info("Configuring basic directory information ...") +this_script_dir = os.path.realpath(os.path.split(__file__)[0]) +base_dir = os.path.realpath(os.path.join(this_script_dir, "..")) +spack_dir = spack.paths.spack_root +logging.info(" ... script directory: {}".format(this_script_dir)) +logging.info(" ... base directory: {}".format(base_dir)) +logging.info(" ... spack directory: {}".format(spack_dir)) + +# Templates for creating compiler modules +COMPILER_TEMPLATES = { + "lmod": os.path.join(this_script_dir, "templates/compiler.lua"), + "tcl": os.path.join(this_script_dir, "templates/compiler"), +} +MPI_TEMPLATES = { + "lmod": os.path.join(this_script_dir, "templates/mpi.lua"), + "tcl": os.path.join(this_script_dir, "templates/mpi"), +} +MODULE_FILE_EXTENSION = { + "lmod": ".lua", + "tcl": "" +} + +SUBSTITUTES_TEMPLATE = { + "MODULELOADS": "", + "MODULEPREREQS": "", + "MODULEPATHS": "", + "CC": "", + "CXX": "", + "F77": "", + "FC": "", + "COMPFLAGS": "", + "ENVVARS": "", + "MPICC": "", + "MPICXX": "", + "MPIF77": "", + "MPIROOT": "", +} + + +COMPILER_TRANSLATION_TABLE = { + "gcc" : "gcc", + "clang" : "llvm", + "intel" : "intel-oneapi-compilers-classic", + "oneapi" : "intel-oneapi-compilers", +} + + +def setenv_command(module_choice, key, value): + if module_choice == "lmod": + return 'setenv("{}", "{}")\n'.format(key, value) + else: + return "setenv {{{}}} {{{}}}\n".format(key, value) + + +def unsetenv_command(module_choice, key, value): + if module_choice == "lmod": + return 'unsetenv("{}", "{}")\n'.format(key, value) + else: + return "unsetenv {{{}}} {{{}}}\n".format(key, value) + + +def append_path_command(module_choice, key, value): + if module_choice == "lmod": + return 'append_path("{}", "{}")\n'.format(key, value) + else: + return "append-path {{{}}} {{{}}}\n".format(key, value) + + +def prepend_path_command(module_choice, key, value): + if module_choice == "lmod": + return 'prepend_path("{}", "{}")\n'.format(key, value) + else: + return "prepend-path {{{}}} {{{}}}\n".format(key, value) + + +def module_load_command(module_choice, module): + if module_choice == "lmod": + return 'load("{}")\n'.format(module) + else: + return """if {{ [ module-info mode load ] && ![ is-loaded {0} ] }} {{ + module load {0} +}}\n""".format( + module + ) + + +def module_prereq_command(module_choice, module): + if module_choice == "lmod": + return 'prereq("{}")\n'.format(module) + else: + return "" + + +def modulepath_prepend_command(module_choice, modulepath): + if module_choice == "lmod": + return 'prepend_path("MODULEPATH", "{}")\n'.format(modulepath) + else: + return "prepend-path {{MODULEPATH}} {{{}}}\n".format(modulepath) + + +def substitute_config_vars(config_str): + """ + Substitute spack-specific and environment variables that may be present + in configuration files. See: + https://spack.readthedocs.io/en/latest/configuration.html#config-file-variables + """ + spack_vars = { + "ENV": ev.active_environment().path, + "SPACK": os.getenv("SPACK_ROOT"), + "TEMPDIR": None, + "USER": os.getenv("HOME"), + "USER_CACHE_PATH": os.path.join(os.getenv("HOME"), ".spack"), + } + + if config_str.startswith("~"): + config_str = config_str.replace("~", os.getenv("HOME")) + + # Get var as it appears in the string (e.g. ${env}), and its name (e.g. env) + matches = re.findall(r"(\$(\w+))|(\${(\w+)})", config_str) + for match in matches: + if match[0]: + pair = (match[0], match[1]) + else: + pair = (match[2], match[3]) + + var_string = pair[0] + var_name = pair[1].upper() + + sub_value = spack_vars[var_name] if spack_vars[var_name] else os.getenv(var_name) + config_str = config_str.replace(var_string, sub_value) + + return config_str + + +def setup_meta_modules(): + # Find currently active spack environment, activate here + logging.info("Configuring active spack environment ...") + env_dir = ev.active_environment().path + if not env_dir: + raise Exception("No active spack environment") + env = spack.environment.Environment(env_dir) + spack.environment.environment.activate(env) + logging.info(" ... environment directory: {}".format(env_dir)) + + # Parse spack main config from environment + logging.info("Parsing spack environment main config ...") + main_config = spack.config.get("config") + install_dir = substitute_config_vars(main_config["install_tree"]["root"]) + + if not os.path.isabs(install_dir): + install_dir = os.path.realpath(os.path.join(env_dir, install_dir)) + else: + install_dir = os.path.realpath(install_dir) + logging.info(" ... install directory: {}".format(install_dir)) + + # Parse spack module config from environment + logging.info("Parsing spack environment modules config ...") + module_config = spack.config.get("modules") + + # Check which modules are used - tcl or lmod (can only be one) + if len(module_config["default"]["enable"]) > 1: + raise Exception("Can use either lmod or tcl modules, not both") + module_choice = module_config["default"]["enable"][0] + logging.info(" ... configured to use {} modules".format(module_choice)) + + # Need to set a few variables when tcl modules are used + if module_choice == "tcl": + module_replace_patterns = ["is-loaded", "module load", "depends-on"] + # sed syntax differs on macOS + if sys.platform == "darwin": + sed_syntax_fix = "''" + else: + sed_syntax_fix = "" + + # Top-level module directory + module_dir = substitute_config_vars(module_config["default"]["roots"][module_choice]) + if not os.path.isabs(module_dir): + module_dir = os.path.realpath(os.path.join(env_dir, module_dir)) + else: + module_dir = os.path.realpath(module_dir) + logging.info(f" ... module directory: {module_dir}") + + # Get all specs and determine compilers and mpi providers + hashes = env.all_hashes() + specs = spack.store.STORE.db.query(hashes=hashes) + q = ProviderIndex(specs=specs, repository=spack.repo.PATH) + + c_providers = q.providers_for("c") + cxx_providers = q.providers_for("cxx") + fortran_providers = q.providers_for("fortran") + compilers = list(set(c_providers + cxx_providers + fortran_providers)) + if not compilers: + raise Exception("No compilers found") + logging.info(f" ... compilers: {compilers}") + + # To remove compiler prefices from tcl modulefiles, we need + # a mock compiler "none@none" for external packages + if module_choice == "tcl": + mock_compiler_list = [x.name+"@"+str(x.version) for x in compilers] + ["none@none"] + logging.debug(f" ... mock_compiler_list: {mock_compiler_list}") + + mpi_providers = q.providers_for("mpi") + if not len(mpi_providers)==1: + raise Exception(f"Expected exactly one MPI provider, but got {mpi_providers}") + logging.info(f" ... mpi_providers: {mpi_providers}") + + # Core compilers is only a valid options for lmod + if module_choice == "lmod": + # Determine core compiler(s) and make sure they are not used + core_compilers = module_config["default"][module_choice]["core_compilers"] + logging.info(" ... core compilers: {}".format(core_compilers)) + # Check that none of the compilers used for the stack is a core compiler + for core_compiler in core_compilers: + if any(core_compiler in x for x in flattened_compiler_list): + raise Exception( + """Not supported: compiler used for environment + is in list of core compilers""" + ) + + # Prepare meta module directory + logging.info("Preparing meta module directory ...") + meta_module_dir = os.path.join(module_dir, "Core") + if not os.path.isdir(meta_module_dir): + os.mkdir(meta_module_dir) + logging.info(" ... meta module directory : {}".format(meta_module_dir)) + + + # DH* 20250730 - DO WE NEED BOTH WAYS TO DETERMINE THE PREFERRED COMPILER? + # MAYBE KEEP FOR NOW UNTIL WE KNOW BETTER WHAT IS NEEDED AND WHAT NOT. + + # Determine the preferred compiler and sort the list of compilers so that + # the preferred compiler comes last. This is so that all other compilers + # populate the MODULEPATHS_SAVE list before the preferred compiler + # takes it and adds it to the stack-COMPILER metamodule. Likewise, we need + # to save the list of compiler substitutions from the preferred compiler + # so that we have access to it when we build the MPI meta module. + try: + preferred_compilers = spack.config.get("packages")["all"]["require"] + except: + raise Exception( + """Unable to detect preferred compiler from environment. + Does the environment have the config entry 'packages:all:require?'""" + ) + if len(preferred_compilers)>1: + raise Exception(f"Invalid value for packages:all:require is {preferred_compilers}") + match = re.search(r'%\[when=%fortran\](\S+)', preferred_compilers[0]) + if match and match.group(1) in COMPILER_TRANSLATION_TABLE.keys(): + preferred_compiler_v1 = COMPILER_TRANSLATION_TABLE[match.group(1)] + else: + raise Exception(f"Invalid value for packages:all:require {preferred_compilers[0]}") + + try: + preferred_compilers = spack.config.get("packages")["fortran"]["require"] + except: + raise Exception( + """Unable to detect preferred compiler from environment. + Does the environment have the config entry 'packages:fortran:require?'""" + ) + if len(preferred_compilers)>1: + raise Exception(f"Invalid value for packages:fortran:require is {preferred_compilers}") + if preferred_compilers[0] in COMPILER_TRANSLATION_TABLE.values(): + preferred_compiler_v2 = preferred_compilers[0] + else: + raise Exception(f"Invalid value for packages:fortran:require {preferred_compilers[0]}") + if preferred_compiler_v1 == preferred_compiler_v2: + preferred_compiler = preferred_compiler_v1 + del preferred_compilers + del preferred_compiler_v1 + del preferred_compiler_v2 + else: + raise Exception(f"Multiple preferred compilers in spack config: {preferred_compiler_v1} vs {preferred_compiler_v2}") + logging.info(" ... preferred compiler: {}".format(preferred_compiler)) + # *DH 20250730 + + # Sort the list using a custom key + def custom_sort_key(entry): + # Return a tuple where the first element is 1 if the entry contains the word, else 0 + # The second element is the entry itself for natural sorting within groups + return (1 if preferred_compiler in entry else 0, entry) + compilers = sorted(compilers, key=custom_sort_key) + + # Create compiler modules + logging.info("Creating compiler modules ...") + + # Collect and save modulepaths for the preferred compiler + MODULEPATHS_SAVE = [] + + # Append modulepath for external specs + modulepath_save = os.path.join(module_dir, "none", "none") + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + logging.info(" ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) + MODULEPATHS_SAVE.append(modulepath_save) + + # Initialize saved substitutes to None (populate for preferred compiler later) + COMPILER_SUBSTITUTES_SAVE = None + + for compiler in compilers: + logging.info(f" ... configuring compiler {compiler.name}@{compiler.version}") + + modulepath_save = os.path.join(module_dir, compiler.name, str(compiler.version)) + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + logging.info(" ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) + MODULEPATHS_SAVE.append(modulepath_save) + + # For tcl modules remove the compiler prefices from the module contents + if module_choice == "tcl": + logging.info( + " ... ... removing compiler prefices from tcl modulefiles in {}".format( + modulepath_save + ) + ) + + for root, ddir, files in os.walk(modulepath_save): + for ffile in files: + filepath = os.path.join(root, ffile) + logging.debug( + " ... ... ... removing compiler prefices in {}".format(filepath) + ) + for pattern in module_replace_patterns: + for tmp_compiler in mock_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") + cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( + pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + + # The remainder of the loop is only needed for the preferred compiler + if not compiler.name in preferred_compiler: + continue + + logging.info( + " ... configuring stack compiler {}@{}".format(compiler.name, compiler.version) + ) + compiler_module_dir = os.path.join(meta_module_dir, "stack-" + compiler.name) + compiler_module_file = os.path.join( + compiler_module_dir, str(compiler.version) + MODULE_FILE_EXTENSION[module_choice] + ) + substitutes = SUBSTITUTES_TEMPLATE.copy() + + if not compiler.external: + raise Exception(f"spack-built compiler {compiler} not yet supported") + + #### Check if the external package has a prefix + ###if compiler.prefix: + ### prefix = compiler.prefix + ### # Workaround for intel-oneapi-mpi. Because of the way this package + ### # is configured in spack, the prefix may be incomplete + ### if compiler.name == "intel-oneapi-compilers" or "intel-oneapi-compilers-classic": + ### if not os.path.isdir(os.path.join(prefix, "bin")): + ### prefix = os.path.join(prefix, "compiler", str(compiler.version)) + ### if not os.path.isdir(os.path.join(prefix, "bin")): + ### raise Exception("Invalid prefix for external compiler: {prefix}") + ###else: + ### prefix = None + + # Compiler environment variables; names are lowercase in spack + substitutes["CC"] = compiler.extra_attributes["compilers"]["c"] + substitutes["CXX"] = compiler.extra_attributes["compilers"]["cxx"] + substitutes["F77"] = compiler.extra_attributes["compilers"]["fortran"] + substitutes["FC"] = compiler.extra_attributes["compilers"]["fortran"] + logging.debug(" ... ... CC : {}".format(substitutes["CC"])) + logging.debug(" ... ... CXX : {}".format(substitutes["CXX"])) + logging.debug(" ... ... F77 : {}".format(substitutes["F77"])) + logging.debug(" ... ... FC' : {}".format(substitutes["FC"])) + + # Compiler flags; names are lowercase in spack + if "flags" in compiler.extra_attributes.keys(): + for flag_name in compiler.extra_attributes["flags"].keys(): + flag_values = compiler["compiler"]["flags"][flag_name] + substitutes["COMPFLAGS"] += setenv_command( + module_choice, flag_name.upper(), flag_values + ) + substitutes["COMPFLAGS"] = substitutes["COMPFLAGS"].rstrip("\n") + logging.debug(" ... ... COMPFLAGS: {}".format(substitutes["COMPFLAGS"])) + + if compiler.external_modules: + raise Exception(f"UNTESTED XYZ MODULES: {compiler.external_modules}") + # Existing non-spack modules to load + for module in compiler.external_modules: + substitutes["MODULELOADS"] += module_load_command(module_choice, module) + substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) + substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") + substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") + logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) + logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) + + # Environment variables; case-sensitive in spack + if "environment" in compiler.extra_attributes.keys(): + # DH* CAN WE SIMPLIFY THIS WITH A LOOP AND A GENERIC FUNCTION? + # append_path + if "append_path" in compiler.extra_attributes["environment"].keys(): + for env_name in compiler.extra_attributes["environment"]["append_path"]: + env_values = compiler.extra_attributes["environment"]["append_path"][env_name] + substitutes["ENVVARS"] += append_path_command( + module_choice, env_name, env_values + ) + # prepend_path + if "prepend_path" in compiler.extra_attributes["environment"].keys(): + for env_name in compiler.extra_attributes["environment"]["prepend_path"]: + env_values = compiler.extra_attributes["environment"]["prepend_path"][env_name] + substitutes["ENVVARS"] += prepend_path_command( + module_choice, env_name, env_values + ) + # set + if "set" in compiler.extra_attributes["environment"].keys(): + for env_name in compiler.extra_attributes["environment"]["set"]: + env_values = compiler.extra_attributes["environment"]["set"][env_name] + substitutes["ENVVARS"] += setenv_command( + module_choice, env_name, env_values + ) + # unset + if "unset" in compiler.extra_attributes["environment"].keys(): + for env_name in compiler.extra_attributes["environment"]["unset"]: + env_values = compiler.extra_attributes["environment"]["unset"][env_name] + substitutes["ENVVARS"] += unsetenv_command( + module_choice, env_name, env_values + ) + # *DH + substitutes["ENVVARS"] = substitutes["ENVVARS"].rstrip("\n") + logging.debug(" ... ... ENVVARS : {}".format(substitutes["ENVVARS"])) + + # Spack compiler module hierarchy - append all saved modulepaths + for modulepath in MODULEPATHS_SAVE: + substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) + substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") + logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) + + # Read compiler template into module_content string + with open(COMPILER_TEMPLATES[module_choice]) as f: + module_content = f.read() + + # Substitute variables in module_content + for key in substitutes.keys(): + module_content = module_content.replace("@{}@".format(key), substitutes[key]) + + # Write compiler lua module + if not os.path.isdir(compiler_module_dir): + os.makedirs(compiler_module_dir) + with open(compiler_module_file, "w") as f: + f.write(module_content) + logging.info(" ... writing {}".format(compiler_module_file)) + # If this is the last compiler in the list (i.e. the preferred compiler), + # then save the substitutes for later use for building the MPI meta module. + if compiler == compilers[-1]: + COMPILER_SUBSTITUTES_SAVE = substitutes + + del MODULEPATHS_SAVE + + # Collect and save modulepaths for MPI with preferred compiler + MODULEPATHS_SAVE = [] + + # Create mpi modules - as of July 2025, only one + # MPI provider is allowed (for the preferred compiler) + for mpi_provider in mpi_providers: + + for compiler in compilers: + logging.info( + " ... configuring stack mpi library {}@{} for compiler {}@{}".format( + mpi_provider.name, mpi_provider.version, compiler.name, compiler.version + ) + ) + + # Spack mpi+compiler module hierarchy + modulepath_save = os.path.join( + module_dir, mpi_provider.name, str(mpi_provider.version), compiler.name, str(compiler.version) + ) + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + logging.info(" ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) + MODULEPATHS_SAVE.append(modulepath_save) + + # For tcl modules remove the compiler/mpi prefices from the module contents + if module_choice == "tcl": + logging.info( + " ... ... removing compiler/mpi prefices from tcl modulefiles in {}".format( + modulepath_save + ) + ) + for root, ddir, files in os.walk(modulepath_save): + for ffile in files: + filepath = os.path.join(root, ffile) + logging.debug( + " ... ... ... removing compiler/mpi prefices in {}".format( + filepath + ) + ) + # First, compiler-only dependent modules + # These can depend on other compilers than the + # compiler this MPI was built with, loop over all + for pattern in module_replace_patterns: + for tmp_compiler in mock_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") + cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( + pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + # Then, compiler+mpi-dependent modules + # By definition, these can only depend on the compiler that + # this MPI was built with - no need to loop over all compilers + for pattern in module_replace_patterns: + for tmp_compiler in mock_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") + cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format( + pattern, + mpi_provider.name, + mpi_provider.version, + tmp_compiler_name, + tmp_compiler_version, + filepath, + sed_syntax_fix, + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + + # The remainder of the loop is only needed for the preferred compiler + if not compiler.name in preferred_compiler: + continue + + # Path and name for mpi module file + mpi_module_dir = os.path.join( + module_dir, compiler.name, str(compiler.version), "stack-" + mpi_provider.name + ) + mpi_module_file = os.path.join( + mpi_module_dir, str(mpi_provider.version).split("-")[0] + MODULE_FILE_EXTENSION[module_choice] + ) + + substitutes = SUBSTITUTES_TEMPLATE.copy() + # + if mpi_provider.external: + # Check if the external package has a prefix + if mpi_provider.prefix: + prefix = mpi_provider.prefix + # Workaround for intel-oneapi-mpi. Because of the way this package + # is configured in spack, the prefix may be incomplete + if mpi_provider.name == "intel-oneapi-mpi": + if not os.path.isdir(os.path.join(prefix, "bin")): + prefix = os.path.join(prefix, "mpi", str(mpi_provider.version)) + if not os.path.isdir(os.path.join(prefix, "bin")): + raise Exception("Invalid prefix for external MPI provider: {prefix}") + # Replace "-" in mpi_provider.name with "_" for _ROOT environment variable + substitutes["MPIROOT"] = setenv_command( + module_choice, mpi_provider.name.replace("-", "_") + "_ROOT", prefix + ) + logging.debug( + " ... ... MPIROOT: {}".format(substitutes["MPIROOT"]).rstrip("\n") + ) + else: + prefix = None + + # Check if the external package has modules + if mpi_provider.external_modules: + raise Exception(f"UNTESTED XYZ MPI MODULES: {mpi_provider.external_modules}") + # Existing non-spack modules to load + for module in mpi_provider.external_modules: + substitutes["MODULELOADS"] += module_load_command( + module_choice, module + ) + substitutes["MODULEPREREQS"] += module_prereq_command( + module_choice, module + ) + substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") + substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") + logging.debug( + " ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"]) + ) + logging.debug( + " ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"]) + ) + # If no modules, then the external package must have a prefix + elif prefix: + # PATH and compiler wrapper environment variables + bindir = os.path.join(prefix, "bin") + if os.path.isdir(bindir): + substitutes["ENVVARS"] += prepend_path_command( + module_choice, "PATH", bindir + ) + # LD_LIBRARY_PATH AND PKG_CONFIG_PATH + libdir = os.path.join(prefix, "lib") + if os.path.isdir(libdir): + substitutes["ENVVARS"] += prepend_path_command( + module_choice, "LD_LIBRARY_PATH", libdir + ) + pkgconfigdir = os.path.join(libdir, "pkgconfig") + if os.path.isdir(pkgconfigdir): + substitutes["ENVVARS"] += prepend_path_command( + module_choice, "PKG_CONFIG_PATH", pkgconfigdir + ) + lib64dir = os.path.join(prefix, "lib64") + if os.path.isdir(lib64dir): + substitutes["ENVVARS"] += prepend_path_command( + module_choice, "LD_LIBRARY_PATH", lib64dir + ) + pkgconfig64dir = os.path.join(lib64dir, "pkgconfig") + if os.path.isdir(pkgconfig64dir): + substitutes["ENVVARS"] += prepend_path_command( + module_choice, "PKG_CONFIG_PATH", pkgconfig64dir + ) + # MANPATH + mandir = os.path.join(prefix, "share/man") + if os.path.isdir(mandir): + substitutes["ENVVARS"] += prepend_path_command( + module_choice, "MANPATH", mandir + ) + # ACLOCAL_PATH + aclocaldir = os.path.join(prefix, "share/aclocal") + if os.path.isdir(aclocaldir): + substitutes["ENVVARS"] += prepend_path_command( + module_choice, "ACLOCAL_PATH", aclocaldir + ) + else: + raise Exception(f"Neither prefix nor modules configured for external MPI provider {mpi_provider}") + else: + # Simple workaround for now - depend on spack building the mpi module + # and load it with the wrapper, don't set any of the other substitutes + module = "{}/{}".format(mpi_provider.name, mpi_provider.version) + substitutes["MODULELOADS"] += module_load_command( + module_choice, module.split("-")[0] + ).rstrip("\n") + substitutes["MODULEPREREQS"] += module_prereq_command( + module_choice, module.split("-")[0] + ).rstrip("\n") + logging.debug( + " ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"]) + ) + logging.debug( + " ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"]) + ) + + # Compiler wrapper environment variables + if mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers": + substitutes["MPICC"] = os.path.join("mpiicx") + substitutes["MPICXX"] = os.path.join("mpiicpx") + if "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and not "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: + substitutes["MPIF77"] = os.path.join("mpiifx") + substitutes["MPIF90"] = os.path.join("mpiifx") + elif not "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: + substitutes["MPIF77"] = os.path.join("mpiifort") + substitutes["MPIF90"] = os.path.join("mpiifort") + else: + raise Exception(f"For {mpi_provider.name}, cannot determine MPI wrapper from FC={COMPILER_SUBSTITUTES_SAVE['FC']}") + elif mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers-classic": + substitutes["MPICC"] = os.path.join("mpiicc") + substitutes["MPICXX"] = os.path.join("mpiicpc") + substitutes["MPIF77"] = os.path.join("mpiifort") + substitutes["MPIF90"] = os.path.join("mpiifort") + else: + substitutes["MPICC"] = os.path.join("mpicc") + substitutes["MPICXX"] = os.path.join("mpic++") + substitutes["MPIF77"] = os.path.join("mpif77") + substitutes["MPIF90"] = os.path.join("mpif90") + # Also set the direct compiler environment variables + substitutes["CC"] = COMPILER_SUBSTITUTES_SAVE["CC"] + substitutes["CXX"] = COMPILER_SUBSTITUTES_SAVE["CXX"] + substitutes["F77"] = COMPILER_SUBSTITUTES_SAVE["F77"] + substitutes["FC"] = COMPILER_SUBSTITUTES_SAVE["FC"] + + # Spack mpi+compiler module hierarchy - append all saved modulepaths + for modulepath in MODULEPATHS_SAVE: + substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) + substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") + logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) + + # Read compiler lua template into module_content string + with open(MPI_TEMPLATES[module_choice]) as f: + module_content = f.read() + + # Substitute variables in module_content + for key in substitutes.keys(): + module_content = module_content.replace( + "@{}@".format(key), substitutes[key] + ) + + # Write mpi lua module + if not os.path.isdir(mpi_module_dir): + os.makedirs(mpi_module_dir) + with open(mpi_module_file, "w") as f: + f.write(module_content) + logging.info(" ... writing {}".format(mpi_module_file)) + + del COMPILER_SUBSTITUTES_SAVE + del MODULEPATHS_SAVE + + logging.info("Metamodule generation completed successfully in {}".format(meta_module_dir)) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler index 78891450f..0929436be 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler @@ -4,20 +4,28 @@ # # conflicts +# old names as of July 2025 - deprecate at some point later conflict stack-aocc conflict stack-apple-clang conflict stack-clang conflict stack-gcc conflict stack-intel conflict stack-oneapi +# new names as of July 2025 +# ??? # conflict stack-aocc +# ??? # conflict stack-apple-clang +conflict stack-llvm +conflict stack-gcc +conflict stack-intel-oneapi-compilers-classic +conflict stack-intel-oneapi-compilers + +# spack compiler module hierarchy +@MODULEPATHS@ # prerequisite modules @MODULELOADS@ @MODULEPREREQS@ -# spack compiler module hierarchy -@MODULEPATHS@ - # compiler environment variables setenv {F77} {@F77@} setenv {FC} {@FC@} diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua index 2f741ddbc..0cc0bc1a2 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua @@ -8,20 +8,28 @@ local pkgNameVer = myModuleFullName() family("MetaCompiler") -- conflicts +-- old names as of July 2025 - deprecate at some point later conflict("stack-aocc") conflict("stack-apple-clang") conflict("stack-clang") conflict("stack-gcc") conflict("stack-intel") conflict("stack-oneapi") +-- new names as of July 2025 +-- ??? # conflict stack-aocc +-- ??? # conflict stack-apple-clang +conflict("stack-llvm") +conflict("stack-gcc") +conflict("stack-intel-oneapi-compilers-classic") +conflict("stack-intel-oneapi-compilers") + +-- spack compiler module hierarchy +@MODULEPATHS@ -- prerequisite modules @MODULELOADS@ @MODULEPREREQS@ --- spack compiler module hierarchy -@MODULEPATHS@ - -- compiler environment variables setenv("F77", "@F77@") setenv("FC", "@FC@") diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi index 88334dfe5..97a7a07dc 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi @@ -9,14 +9,15 @@ conflict stack-intel-oneapi-mpi conflict stack-cray-mpich conflict stack-mpich conflict stack-mpt +conflict stack-openmpi + +# spack compiler module hierarchy +@MODULEPATHS@ # prerequisite modules @MODULELOADS@ @MODULEPREREQS@ -# spack compiler module hierarchy -@MODULEPATHS@ - # mpi wrapper environment variables setenv {MPICC} {@MPICC@} setenv {MPICXX} {@MPICXX@} diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua index 4f357d6e7..7e4099ed2 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua @@ -13,14 +13,15 @@ conflict("stack-intel-oneapi-mpi") conflict("stack-cray-mpich") conflict("stack-mpich") conflict("stack-mpt") +conflict("stack-openmpi") + +-- spack compiler module hierarchy +@MODULEPATHS@ -- prerequisite modules @MODULELOADS@ @MODULEPREREQS@ --- spack compiler module hierarchy -@MODULEPATHS@ - -- mpi wrapper environment variables setenv("MPICC", "@MPICC@") setenv("MPICXX", "@MPICXX@") diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/python b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/python deleted file mode 100644 index 81d1194c9..000000000 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/python +++ /dev/null @@ -1,24 +0,0 @@ -#%Module -# -# Python meta module -# - -# conflicts -conflict stack-python - -# prerequisite modules -@MODULELOADS@ -@MODULEPREREQS@ - -# compiler flags and other environment variables -@COMPFLAGS@ -@ENVVARS@ - -# python root environment variable -@PYTHONROOT@ - -proc ModulesHelp {} { - puts -nonewline stderr "This modulefile defines the python meta module for spack-stack" -} - -module-whatis "Python and module access" diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/python.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/python.lua deleted file mode 100644 index fbeafea3f..000000000 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/python.lua +++ /dev/null @@ -1,28 +0,0 @@ -help([[ -]]) - -local pkgName = myModuleName() -local pkgVersion = myModuleVersion() -local pkgNameVer = myModuleFullName() - -family("MetaPython") - --- conflicts -conflict("stack-python") - --- prerequisite modules -@MODULELOADS@ -@MODULEPREREQS@ - --- compiler flags and other environment variables -@COMPFLAGS@ -@ENVVARS@ - --- python root environment variable -@PYTHONROOT@ - --- module show info -whatis("Name: " .. pkgName) -whatis("Version: " .. pkgVersion) -whatis("Category: interpreter") -whatis("Description: " .. pkgName .. " interpreter and module access") From 09e1236b4dbef89bedaabdf236045bcf1667d9b7 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 30 Jul 2025 13:13:52 -0600 Subject: [PATCH 009/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index beafe0546..2da6975b2 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit beafe0546edf678a526ce697c701e1756b2f86c6 +Subproject commit 2da6975b236768b8dd86c65f874750093e49d4e7 From 399ed303f88df54f053dcf9fdffff9e3b710f922 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 31 Jul 2025 14:54:59 -0600 Subject: [PATCH 010/163] [skip ci] Work in progress: common config for gcc and oneapi --- configs/common/packages.yaml | 215 ++++++++++-------- configs/common/packages_oneapi.yaml | 38 +++- repos/builtin | 2 +- .../packages/neptune_env/package.py | 4 + .../packages/neptune_python_env/package.py | 10 +- 5 files changed, 158 insertions(+), 111 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index ea841dd9a..af225287e 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -21,13 +21,16 @@ packages: # 1.85 incompatible with ecflow@5.11.4 - use latest 'good version' boost: require: - - '@1.84 ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden' + - '@1.84' + - '~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden' bufr: require: - - '@12.1.0 +python' + - '@12.1.0 + - '+python' bufr-query: require: - - '@0.0.4 +python' + - '@0.0.4 + - '+python' cairo: require: '+pic' cdo: @@ -61,13 +64,16 @@ packages: - '+netcdf +png +tools' ecflow: require: - - '@5.11.4 +ui' + - '@5.11.4' + - '+ui' eckit: require: - - '@1.28.3 linalg=eigen,lapack compression=lz4,bzip2' + - '@1.28.3' + - 'linalg=eigen,lapack compression=lz4,bzip2' ecmwf-atlas: require: - - '@0.40.0 +fckit +trans +tesselation +fftw' + - '@0.40.0' + - '+fckit +trans +tesselation +fftw' ectrans: require: - '@1.5.0' @@ -79,27 +85,28 @@ packages: # Also, check the acorn and derecho site configs which have esmf modifications. esmf: require: - - '~xerces ~pnetcdf +shared +external-parallelio' + - '~xerces ~pnetcdf +shared +external-parallelio +python' - any_of: ['@=8.6.1 snapshot=none', '@=8.8.0 snapshot=none', '@=8.9.0b12 snapshot=b12'] - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] when: "%intel" message: "Extra ESMF compile options for Intel" - - any_of: ['+python'] - when: "%gcc" - - any_of: ['+python'] - when: "%intel" - - any_of: ['+python'] - when: "%oneapi" - - any_of: ['~python'] - when: "%clang" - message: "Disable python variant for ESMF because mpi4py doesn't build with clang" + #- any_of: ['+python'] + # when: "%gcc" + #- any_of: ['+python'] + # when: "%intel" + #- any_of: ['+python'] + # when: "%oneapi" + #- any_of: ['~python'] + # when: "%clang" + # message: "Disable python variant for ESMF because mpi4py doesn't build with clang" # To avoid duplicate packages (concretizer bug?) expat: require: - '~libbsd' fckit: require: - - '@0.13.2 +eckit' + - '@0.13.2' + - '+eckit' fftw: require: - '@3.3.10' @@ -129,10 +136,9 @@ packages: g2tmpl: require: - '@1.13.0' - # DH* NEEDED??? - #gcc-runtime: - # require: - # - '%gcc' + gcc-runtime: + require: + - '%gcc' gfsio: require: - '@1.4.2' @@ -142,15 +148,15 @@ packages: # Note: Uncommenting this entry will break # the container builds. #require: '@2.11.0' - # https://github.com/spack/spack/issues/49358 - # https://github.com/spack/spack/pull/44323 - glib: - require: - - '@:2.78.3' - # To avoid duplicate packages being built - gmake: - require: - - one_of: ['@=4.2.1', '@=4.4.1'] + ## https://github.com/spack/spack/issues/49358 + ## https://github.com/spack/spack/pull/44323 + #glib: + # require: + # - '@:2.78.3' + ## To avoid duplicate packages being built + #gmake: + # require: + # - one_of: ['@=4.2.1', '@=4.4.1'] grib-util: require: - '@1.4.0' @@ -160,12 +166,13 @@ packages: gsi-ncdiag: require: - '@1.1.2' - gsl-lite: - require: - - '@0.37.0' + #gsl-lite: + # require: + # - '@0.37.0' hdf: require: - - '@4.2.15 +external-xdr ~fortran ~netcdf' + - '@4.2.15' + - '+external-xdr ~fortran ~netcdf' hdf5: require: - '@1.14.5' @@ -174,27 +181,35 @@ packages: hdf-eos2: require: - '@2.20v1.00' + intel-oneapi-compilers-runtime: + require: + - '%oneapi' + intel-oneapi-compilers-classic-runtime: + require: + - '%intel' ip: require: - - '@5.4.0 precision=4,d,8' + - '@5.4.0' + - 'precision=4,d,8' ip2: require: - '@1.1.2' - jasper: - require: - - '@2.0.32' + #jasper: + # require: + # - '@2.0.32' jedi-cmake: require: - '@1.4.0' landsfcutil: require: - '@2.4.2' - libjpeg-turbo: - require: - - '@2.1.0' + #libjpeg-turbo: + # require: + # - '@2.1.0' libpng: require: - - '@1.6.37 +pic' + - '@1.6.37' + - '+pic' # libtirpc +gssapi doesn't build with oneapi@2025 # https://github.com/JCSDA/spack-stack/issues/1441 libtirpc: @@ -206,13 +221,12 @@ packages: - '@4.15.3:' mapl: require: - - '@2.53 ~shared ~f2py +pflogger' - #variants: '+pflogger' + - '@2.53' + - '~shared ~f2py +pflogger' met: require: - '@12.0.1' - - '+python' - - '+grib2' + - '+python +grib2' metplus: require: - '@6.0.0' @@ -240,19 +254,21 @@ packages: require: - '@1.1.2' netcdf-c: - # If using 4.9.1, turn off byterange variant to fix compile error: ~byterange require: - - '@4.9.2 +dap +mpi ~parallel-netcdf ~szip +parallel_tests build_system=autotools' + - '@4.9.2' + - '+dap +mpi ~parallel-netcdf ~szip +parallel_tests build_system=autotools' netcdf-cxx4: require: - '@4.3.1' netcdf-fortran: require: - - '@4.6.1 +parallel_tests' + - '@4.6.1' + - '+parallel_tests' # ninja - when adding information here, also check Discover site config odc: require: - - '@1.6.1 ~fortran' + - '@1.6.1' + - '~fortran' openmpi: require: - '~internal-hwloc +two_level_namespace' @@ -265,6 +281,7 @@ packages: parallelio: require: - '@2.6.2' + # DH* CAN WE UPDATE? NEED A NEW SHARED-INTEL PATCH? parallel-netcdf: require: - '@1.12.3' @@ -285,68 +302,71 @@ packages: # are removed, which itself is problematic. python: require: - - '@3.11 ~crypt' + - '@3.11' + - '~crypt' py-cartopy: require: - '+plotting' - # Pin py-cython to avoid duplicate packages - py-cython: - require: - - '@3' - # As long as we need the "delayed" variant for py-dask, - # we are restricted to version 2021.03.0 (last one that has it) - # and newer than 2021.01.0 because of these issues: - # https://github.com/JCSDA/spack-stack/issues/1216 - # https://github.com/pydata/xarray/issues/8917 - py-dask: - require: - - '@2021.3.0' - # To avoid duplicate packages - py-flit-core: - require: - - '@3.8.0' + ## Pin py-cython to avoid duplicate packages + #py-cython: + # require: + # - '@3' + ## As long as we need the "delayed" variant for py-dask, + ## we are restricted to version 2021.03.0 (last one that has it) + ## and newer than 2021.01.0 because of these issues: + ## https://github.com/JCSDA/spack-stack/issues/1216 + ## https://github.com/pydata/xarray/issues/8917 + #py-dask: + # require: + # - '@2021.3.0' + ## To avoid duplicate packages + #py-flit-core: + # require: + # - '@3.8.0' py-h5py: require: - '~mpi' # https://github.com/JCSDA/spack-stack/issues/1276 - py-matplotlib: - require: - - '@3.7.4' + #py-matplotlib: + # require: + # - '@3.7.4' # https://github.com/Unidata/netcdf4-python/issues/1389 # https://github.com/Unidata/netcdf4-python/issues/1389#issuecomment-2599760051 py-netcdf4: require: - - '@1.7.1 ~mpi' - # To avoid duplicate packages - py-numpy: - require: - - '@1.26' - # To avoid duplicate packages - py-ruamel-yaml: - require: - - '@0.17.16' - # Pin the py-setuptools version to avoid duplicate Python packages - py-setuptools: - require: - - '@69' - # When making changes here, also check the packages.yaml for atlantis, nautilus, blackpearl + - '@1.7.2' + - '~mpi' + ## To avoid duplicate packages + #py-numpy: + # require: + # - '@1.26' + ## To avoid duplicate packages + #py-ruamel-yaml: + # require: + # - '@0.17.16' + ## Pin the py-setuptools version to avoid duplicate Python packages + #py-setuptools: + # require: + # - '@69' + ## When making changes here, also check the packages.yaml for atlantis, nautilus, blackpearl py-torch: require: - '+custom-protobuf ~mkldnn' - # To avoid duplicate packages - py-urllib3: - require: - - '@1.26.12' - # To avoid duplicate packages - py-versioneer: - require: - - '@0.28' + ## To avoid duplicate packages + #py-urllib3: + # require: + # - '@1.26.12' + ## To avoid duplicate packages + #py-versioneer: + # require: + # - '@0.28' qt: require: - '@5' scotch: require: - - '@7.0.4 +mpi+metis~shared~threads~mpi_thread+noarch+esmumps' + - '@7.0.4' + - '+mpi+metis~shared~threads~mpi_thread+noarch+esmumps' sfcio: require: - '@1.4.2' @@ -358,7 +378,8 @@ packages: - '@2.3.3' sp: require: - - '@2.5.0 precision=4,d,8' + - '@2.5.0' + - 'precision=4,d,8' udunits: require: - '@2.2.28' @@ -372,7 +393,8 @@ packages: # Need extradeps for grib-utils, enable by default to avoid duplicate packages w3emc: require: - - '@2.10.0 precision=4,d,8 +extradeps' + - '@2.10.0' + - 'precision=4,d,8 +extradeps' w3nco: require: - '@2.4.1' @@ -385,4 +407,5 @@ packages: - '@1.2.0' zstd: require: - - '@1.5.6 +programs' + - '@1.5.7' + - '+programs' diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index 49dccd083..01b46b3d3 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -22,42 +22,58 @@ packages: blas: [intel-oneapi-mkl] fftw-api: [intel-oneapi-mkl] lapack: [intel-oneapi-mkl] - ectrans: - require: - - '+mkl ~fftw' - gsibec: - require: - - '+mkl' - py-numpy: - require: - - '^intel-oneapi-mkl' antlr: require: - '%gcc' bison: require: - '%gcc' + boost: + require: + - '%oneapi' cmake: require: - '%gcc' + ectrans: + require: + - '+mkl ~fftw' gmake: require: - '%gcc' - libmd: + gsibec: require: - - '%gcc' + - '+mkl' libbsd: require: - '%gcc' + libmd: + require: + - '%gcc' met: require: - '+shared-intel' + netcdf-c: + require: + - '%oneapi' + netcdf-cxx: + require: + - '%oneapi' nco: require: - '%gcc' + p4est: + require: + - '%oneapi' parallel-netcdf: require: - '+shared-intel' + python: + require: + - '%oneapi' + py-numpy: + require: + - '%oneapi' + - '^intel-oneapi-mkl' qt: require: - '%gcc' diff --git a/repos/builtin b/repos/builtin index 2da6975b2..eacd94be2 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 2da6975b236768b8dd86c65f874750093e49d4e7 +Subproject commit eacd94be26e036e5f28d4db6cde8d496772f207e diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py index a7ee62d0f..9f7f71f9e 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py @@ -23,6 +23,10 @@ class NeptuneEnv(BundlePackage): variant("espc", default=False, description="Build ESPC dependencies") variant("debug", default=False, description="Build debug version of selected dependencies") + #depends_on("c", type="run") + #depends_on("cxx", type="run") + #depends_on("fortran", type="run") + depends_on("base-env", type="run") depends_on("blas", type="run") diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py index 605ac0b59..2f4d36cd3 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py @@ -22,6 +22,10 @@ class NeptunePythonEnv(BundlePackage): variant("xnrl", default=False, description="Build non-pulic XNRL") + #depends_on("c", type="run") + #depends_on("cxx", type="run") + #depends_on("fortran", type="run") + depends_on("neptune-env", type="run") # Enable the Python variant for ESMF depends_on("esmf +python", type="run") @@ -36,12 +40,12 @@ class NeptunePythonEnv(BundlePackage): depends_on("py-pyyaml", type="run") depends_on("py-regionmask", type="run") depends_on("py-scipy", type="run") - depends_on("py-xarray", type="run") + #depends_on("py-xarray", type="run") depends_on("py-pytest", type="run") depends_on("py-fortranformat", type="run") - depends_on("met", type="run") - depends_on("metplus", type="run") + #depends_on("met", type="run") + #depends_on("metplus", type="run") with when("+xnrl"): depends_on("py-xnrl", type="run") From 7680f0ac51acd077826dfdce87ca8881058cb9e2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 31 Jul 2025 15:17:07 -0600 Subject: [PATCH 011/163] [skip ci] Atlantis: packages_gcc.yaml -> packages_gcc-13.4.0.yaml --- .../{compilers.yaml => compilers.yaml.old} | 0 .../tier1/atlantis/packages_gcc-13.4.0.yaml | 28 +++++++++++++++++++ .../sites/tier1/atlantis/packages_gcc.yaml | 15 ---------- 3 files changed, 28 insertions(+), 15 deletions(-) rename configs/sites/tier1/atlantis/{compilers.yaml => compilers.yaml.old} (100%) create mode 100644 configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml delete mode 100644 configs/sites/tier1/atlantis/packages_gcc.yaml diff --git a/configs/sites/tier1/atlantis/compilers.yaml b/configs/sites/tier1/atlantis/compilers.yaml.old similarity index 100% rename from configs/sites/tier1/atlantis/compilers.yaml rename to configs/sites/tier1/atlantis/compilers.yaml.old diff --git a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml new file mode 100644 index 000000000..d0faf0766 --- /dev/null +++ b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml @@ -0,0 +1,28 @@ +packages: + mpi: + buildable: False + openmpi: + buildable: False + externals: + - spec: openmpi@4.1.5 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm %gcc@=13.4.0 + # Note: slurm must be loaded before openmpi on Atlantis + modules: + - slurm + - openmpi/mlnx/gcc/64/4.1.5a1 + gcc: + externals: + - spec: gcc@13.4.0 languages:='c,c++,fortran' + prefix: /gpfs/neptune/spack-stack/gcc-13.4.0 + extra_attributes: + compilers: + c: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/gcc + cxx: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/g++ + fortran: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/gfortran + extra_rpaths: + # For compiling parallel-netcdf; Bright must have removed + # some symbolic links required for OpenMPI 4.1.5a1 ... + - /gpfs/neptune/spack-stack/openmpi-4.1.5-fix-lib + gcc-runtime: + externals: + - spec: gcc-runtime@13.4.0%gcc@13.4.0 + prefix: /gpfs/neptune/spack-stack/gcc-13.4.0 diff --git a/configs/sites/tier1/atlantis/packages_gcc.yaml b/configs/sites/tier1/atlantis/packages_gcc.yaml deleted file mode 100644 index 91118e459..000000000 --- a/configs/sites/tier1/atlantis/packages_gcc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -packages: - all: - compiler:: [gcc@13.4.0] - providers: - mpi:: [openmpi@4.1.5] - mpi: - buildable: False - openmpi: - buildable: False - externals: - - spec: openmpi@4.1.5 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm %gcc@=13.4.0 - # Note: slurm must be loaded before openmpi on Atlantis - modules: - - slurm - - openmpi/mlnx/gcc/64/4.1.5a1 From 2832594d5ee3fd550fa1945184a379be8b19d89c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 31 Jul 2025 15:21:24 -0600 Subject: [PATCH 012/163] [skip ci] update configs/templates/neptune-dev/spack.yaml --- configs/templates/neptune-dev/spack.yaml | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 17cb0fbab..32a73ca36 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -5,23 +5,10 @@ spack: view: false include: [] - definitions: - - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - - packages: - - neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 - - neptune-env +debug +espc ^esmf@=8.9.0b12 snapshot=b12 - - neptune-python-env +xnrl ^neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 - specs: - - matrix: - - [$packages] - - [$compilers] - exclude: - # Skip neptune-python-env with Intel Classic due to problems - # with new versions of py-numpy, py-scipy, ... - - neptune-python-env%intel - # Same for LLVM compilers - - neptune-python-env%clang + - neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 + - neptune-env +debug +espc ^esmf@=8.9.0b12 snapshot=b12 + - neptune-python-env +xnrl ^neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 packages: # Avoid duplicate builds of parallelio +pnetcdf and ~pnetcdf parallelio: From 16587c21f18342090896d85d588b10eaaa2a7581 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 31 Jul 2025 15:30:32 -0600 Subject: [PATCH 013/163] Bug fixes in configs/common/packages.yaml and configs/sites/tier1/atlantis/packages.yaml --- configs/common/packages.yaml | 7 ++++--- configs/sites/tier1/atlantis/packages.yaml | 22 +++++++++------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index af225287e..ec588a488 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -25,14 +25,15 @@ packages: - '~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden' bufr: require: - - '@12.1.0 + - '@12.1.0' - '+python' bufr-query: require: - - '@0.0.4 + - '@0.0.4' - '+python' cairo: - require: '+pic' + require: + - '+pic' cdo: require: - any_of: ['@2.3.0 ~openmp'] diff --git a/configs/sites/tier1/atlantis/packages.yaml b/configs/sites/tier1/atlantis/packages.yaml index 859ced953..eef70a0e9 100644 --- a/configs/sites/tier1/atlantis/packages.yaml +++ b/configs/sites/tier1/atlantis/packages.yaml @@ -1,22 +1,14 @@ packages: -# Modification of common packages - # Use system zlib instead of spack-built zlib-ng - all: - providers: - zlib-api:: [zlib] + # Modification of common packages + # Prevent building zlib, zlib-ng zlib-api: - buildable: False - zlib: - buildable: False - externals: - - spec: zlib@1.2.11 - prefix: /usr + buildable: false # Pin py-torch to 2.5.1 to avoid build errors with 2.6.0 py-torch: require: - - '@2.5.1' + - '@2.5.1' -# All other packages listed alphabetically + # All other packages listed alphabetically autoconf: externals: - spec: autoconf@2.69 @@ -116,3 +108,7 @@ packages: externals: - spec: wget@1.19.5 prefix: /usr + zlib: + externals: + - spec: zlib@1.2.11 + prefix: /usr From 04a10e7af08fde4273bd4aba6d37a5b7d22e5ea1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 1 Aug 2025 07:52:40 -0700 Subject: [PATCH 014/163] Update spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py for lmod --- .../lib/jcsda-emc/spack-stack/stack/meta_modules.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 5388313d6..ca66f2fa4 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -211,10 +211,11 @@ def setup_meta_modules(): logging.info(f" ... compilers: {compilers}") # To remove compiler prefices from tcl modulefiles, we need - # a mock compiler "none@none" for external packages - if module_choice == "tcl": - mock_compiler_list = [x.name+"@"+str(x.version) for x in compilers] + ["none@none"] - logging.debug(f" ... mock_compiler_list: {mock_compiler_list}") + # a mock compiler "none@none" for external packages. Also + # need a list for lmod to check core compiler + #if module_choice == "tcl": + mock_compiler_list = [x.name+"@"+str(x.version) for x in compilers] + ["none@none"] + logging.debug(f" ... mock_compiler_list: {mock_compiler_list}") mpi_providers = q.providers_for("mpi") if not len(mpi_providers)==1: @@ -228,7 +229,7 @@ def setup_meta_modules(): logging.info(" ... core compilers: {}".format(core_compilers)) # Check that none of the compilers used for the stack is a core compiler for core_compiler in core_compilers: - if any(core_compiler in x for x in flattened_compiler_list): + if any(core_compiler in x for x in mock_compiler_list): raise Exception( """Not supported: compiler used for environment is in list of core compilers""" @@ -363,7 +364,6 @@ def custom_sort_key(entry): # Use existing modules for external mpi providers; otherwise, use spack-built module if compiler.external and compiler.external_modules: - raise Exception(f"UNTESTED XYZ MODULES: {compiler.external_modules}") for module in compiler.external_modules: substitutes["MODULELOADS"] += module_load_command(module_choice, module) substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) @@ -542,7 +542,6 @@ def custom_sort_key(entry): substitutes = SUBSTITUTES_TEMPLATE.copy() # Use existing modules for external mpi providers; otherwise, use spack-built module if mpi_provider.external and mpi_provider.external_modules: - raise Exception(f"UNTESTED XYZ MPI MODULES: {mpi_provider.external_modules}") for module in mpi_provider.external_modules: substitutes["MODULELOADS"] += module_load_command(module_choice, module) substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) From f009c4b6f791cdb01d75b13f4b717130b2df8b8d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 1 Aug 2025 09:08:36 -0600 Subject: [PATCH 015/163] Update configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml, repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py, repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py --- .../sites/tier1/atlantis/packages_gcc-13.4.0.yaml | 2 ++ .../spack_stack/packages/neptune_env/package.py | 12 ++++-------- .../packages/neptune_python_env/package.py | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml index d0faf0766..632b6ba2e 100644 --- a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml +++ b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml @@ -13,6 +13,8 @@ packages: externals: - spec: gcc@13.4.0 languages:='c,c++,fortran' prefix: /gpfs/neptune/spack-stack/gcc-13.4.0 + modules: + - gcc/13.4.0 extra_attributes: compilers: c: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/gcc diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py index 9f7f71f9e..1ea89b4fe 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py @@ -23,10 +23,6 @@ class NeptuneEnv(BundlePackage): variant("espc", default=False, description="Build ESPC dependencies") variant("debug", default=False, description="Build debug version of selected dependencies") - #depends_on("c", type="run") - #depends_on("cxx", type="run") - #depends_on("fortran", type="run") - depends_on("base-env", type="run") depends_on("blas", type="run") @@ -38,14 +34,14 @@ class NeptuneEnv(BundlePackage): depends_on("p4est", type="run") depends_on("w3emc", type="run") depends_on("ip", type="run") - depends_on("esmf ~debug", type="run", when="~debug") - depends_on("esmf +debug", type="run", when="+debug") + depends_on("esmf@8.9.0b12 ~debug ~python snapshot=b12", type="run", when="~debug") + depends_on("esmf@8.9.0b12 +debug ~python snapshot=b12", type="run", when="+debug") depends_on("nco", type="run") depends_on("mct", type="run") with when("+espc"): - depends_on("fftw", type="build") - depends_on("netlib-lapack", type="build") + depends_on("fftw", type="run") + depends_on("netlib-lapack", type="run") # Basic Python dependencies that are always needed depends_on("py-f90nml", type="run") diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py index 2f4d36cd3..21e41d953 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py @@ -47,7 +47,7 @@ class NeptunePythonEnv(BundlePackage): #depends_on("met", type="run") #depends_on("metplus", type="run") - with when("+xnrl"): - depends_on("py-xnrl", type="run") + #with when("+xnrl"): + # depends_on("py-xnrl", type="run") # There is no need for install() since there is no code. From 0ee6d344e486db3ffce9ea33f3fc4fd6193bb1c4 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 1 Aug 2025 11:33:39 -0600 Subject: [PATCH 016/163] Incremental improvements in spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py --- .../spack-stack/stack/meta_modules.py | 199 ++++++++++-------- 1 file changed, 117 insertions(+), 82 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index ca66f2fa4..d0e32fd13 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -90,6 +90,27 @@ def prepend_path_command(module_choice, key, value): return "prepend-path {{{}}} {{{}}}\n".format(key, value) +def envmod_command(module_choice, action, env_name, env_values): + if action == "set": + module_action = "setenv" + elif action == "unset": + module_action = "unsetenv" + elif action == "append_path": + if module_choice == "lmod": + module_action = "append_path" + else: + module_action = "append-path" + elif action == "prepend_path": + if module_choice == "lmod": + module_action = "prepend_path" + else: + module_action = "prepend-path" + if module_choice == "lmod": + return f'{module_action}("{env_name}", "{env_values}")\n'#.format(module_action, env_name, env_values) + else: + return f"{module_action} {{{env_name}}} {{{env_values}}}\n"#.format(module_action, env_name, env_values) + + def module_load_command(module_choice, module): if module_choice == "lmod": return 'load("{}")\n'.format(module) @@ -149,6 +170,48 @@ def substitute_config_vars(config_str): return config_str +def remove_compiler_prefices_from_tcl_modulefiles(modulepath, compiler_list, mpi_provider): + """Remove compiler and mpi prefices from tcl modulefiles in modulepath""" + logging.info(f" ... ... removing compiler/mpi prefices from tcl modulefiles in {modulepath}") + module_replace_patterns = ["is-loaded", "module load", "depends-on"] + # sed syntax differs on macOS + if sys.platform == "darwin": + sed_syntax_fix = "''" + else: + sed_syntax_fix = "" + for root, ddir, files in os.walk(modulepath): + for ffile in files: + filepath = os.path.join(root, ffile) + logging.debug(f" ... ... ... {filepath}") + # First, compiler-only dependent modules + # These can depend on other compilers than the + # compiler this MPI was built with, loop over all + for pattern in module_replace_patterns: + for compiler in compiler_list: + (compiler_name, compiler_version) = compiler.split("@") + cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( + pattern, compiler_name, compiler_version, filepath, sed_syntax_fix + ) + status = os.system(cmd) + if not status == 0: + raise Exception(f"Error while calling '{cmd}'") + # If mpi_provider is not None, also do compiler+mpi-dependent modules + if not mpi_provider: + continue + cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format( + pattern, + mpi_provider.name, + mpi_provider.version, + compiler_name, + compiler_version, + filepath, + sed_syntax_fix, + ) + status = os.system(cmd) + if not status == 0: + raise Exception(f"Error while calling '{cmd}'") + + def setup_meta_modules(): # Find currently active spack environment, activate here logging.info("Configuring active spack environment ...") @@ -180,14 +243,14 @@ def setup_meta_modules(): module_choice = module_config["default"]["enable"][0] logging.info(" ... configured to use {} modules".format(module_choice)) - # Need to set a few variables when tcl modules are used - if module_choice == "tcl": - module_replace_patterns = ["is-loaded", "module load", "depends-on"] - # sed syntax differs on macOS - if sys.platform == "darwin": - sed_syntax_fix = "''" - else: - sed_syntax_fix = "" + ## Need to set a few variables when tcl modules are used + #if module_choice == "tcl": + # module_replace_patterns = ["is-loaded", "module load", "depends-on"] + # # sed syntax differs on macOS + # if sys.platform == "darwin": + # sed_syntax_fix = "''" + # else: + # sed_syntax_fix = "" # Top-level module directory module_dir = substitute_config_vars(module_config["default"]["roots"][module_choice]) @@ -214,8 +277,8 @@ def setup_meta_modules(): # a mock compiler "none@none" for external packages. Also # need a list for lmod to check core compiler #if module_choice == "tcl": - mock_compiler_list = [x.name+"@"+str(x.version) for x in compilers] + ["none@none"] - logging.debug(f" ... mock_compiler_list: {mock_compiler_list}") + compiler_list = [x.name+"@"+str(x.version) for x in compilers] + ["none@none"] + logging.debug(f" ... compiler_list: {compiler_list}") mpi_providers = q.providers_for("mpi") if not len(mpi_providers)==1: @@ -229,7 +292,7 @@ def setup_meta_modules(): logging.info(" ... core compilers: {}".format(core_compilers)) # Check that none of the compilers used for the stack is a core compiler for core_compiler in core_compilers: - if any(core_compiler in x for x in mock_compiler_list): + if any(core_compiler in x for x in compiler_list): raise Exception( """Not supported: compiler used for environment is in list of core compilers""" @@ -300,18 +363,25 @@ def custom_sort_key(entry): # Create compiler modules logging.info("Creating compiler modules ...") + # Initialize saved substitutes to None (populate for preferred compiler later) + COMPILER_SUBSTITUTES_SAVE = None + # Collect and save modulepaths for the preferred compiler MODULEPATHS_SAVE = [] - # Append modulepath for external specs + # Append modulepath for external specs and for specs without compiler dependencies modulepath_save = os.path.join(module_dir, "none", "none") if not os.path.isdir(modulepath_save): os.makedirs(modulepath_save) logging.info(" ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) MODULEPATHS_SAVE.append(modulepath_save) - - # Initialize saved substitutes to None (populate for preferred compiler later) - COMPILER_SUBSTITUTES_SAVE = None + # For tcl modules remove the compiler prefices from the module contents + if module_choice == "tcl": + remove_compiler_prefices_from_tcl_modulefiles( + modulepath_save, + compiler_list, + mpi_provider = None + ) for compiler in compilers: logging.info(f" ... configuring compiler {compiler.name}@{compiler.version}") @@ -324,28 +394,12 @@ def custom_sort_key(entry): # For tcl modules remove the compiler prefices from the module contents if module_choice == "tcl": - logging.info( - " ... ... removing compiler prefices from tcl modulefiles in {}".format( - modulepath_save - ) + remove_compiler_prefices_from_tcl_modulefiles( + modulepath_save, + compiler_list, + mpi_provider = None ) - for root, ddir, files in os.walk(modulepath_save): - for ffile in files: - filepath = os.path.join(root, ffile) - logging.debug( - " ... ... ... removing compiler prefices in {}".format(filepath) - ) - for pattern in module_replace_patterns: - for tmp_compiler in mock_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") - cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( - pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) - # The remainder of the loop is only needed for the preferred compiler if not compiler.name in preferred_compiler: continue @@ -398,6 +452,15 @@ def custom_sort_key(entry): # Environment variables; case-sensitive in spack if "environment" in compiler.extra_attributes.keys(): + for action in compiler.extra_attributes["environment"].keys(): + for env_name in compiler.extra_attributes["environment"][action]: + env_values = compiler.extra_attributes["environment"][action][env_name] + substitutes["ENVVARS"] += envmod_command( + module_choice, + action, + env_name, + env_values + ) # DH* CAN WE SIMPLIFY THIS WITH A LOOP AND A GENERIC FUNCTION? # append_path if "append_path" in compiler.extra_attributes["environment"].keys(): @@ -456,8 +519,6 @@ def custom_sort_key(entry): if compiler == compilers[-1]: COMPILER_SUBSTITUTES_SAVE = substitutes - del MODULEPATHS_SAVE - # Collect and save modulepaths for MPI with preferred compiler MODULEPATHS_SAVE = [] @@ -465,6 +526,21 @@ def custom_sort_key(entry): # MPI provider is allowed (for the preferred compiler) for mpi_provider in mpi_providers: + # Append modulepath for external specs and specs without compiler dependencies + modulepath_save = os.path.join(module_dir, mpi_provider.name, str(mpi_provider.version), "none", "none") + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + logging.info(" ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) + MODULEPATHS_SAVE.append(modulepath_save) + + # For tcl modules remove the compiler/mpi prefices from the module contents + if module_choice == "tcl": + remove_compiler_prefices_from_tcl_modulefiles( + modulepath_save, + compiler_list, + mpi_provider = mpi_provider + ) + for compiler in compilers: logging.info( " ... configuring stack mpi library {}@{} for compiler {}@{}".format( @@ -483,49 +559,11 @@ def custom_sort_key(entry): # For tcl modules remove the compiler/mpi prefices from the module contents if module_choice == "tcl": - logging.info( - " ... ... removing compiler/mpi prefices from tcl modulefiles in {}".format( - modulepath_save - ) + remove_compiler_prefices_from_tcl_modulefiles( + modulepath_save, + compiler_list, + mpi_provider = mpi_provider ) - for root, ddir, files in os.walk(modulepath_save): - for ffile in files: - filepath = os.path.join(root, ffile) - logging.debug( - " ... ... ... removing compiler/mpi prefices in {}".format( - filepath - ) - ) - # First, compiler-only dependent modules - # These can depend on other compilers than the - # compiler this MPI was built with, loop over all - for pattern in module_replace_patterns: - for tmp_compiler in mock_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") - cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( - pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) - # Then, compiler+mpi-dependent modules - # By definition, these can only depend on the compiler that - # this MPI was built with - no need to loop over all compilers - for pattern in module_replace_patterns: - for tmp_compiler in mock_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") - cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format( - pattern, - mpi_provider.name, - mpi_provider.version, - tmp_compiler_name, - tmp_compiler_version, - filepath, - sed_syntax_fix, - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) # The remainder of the loop is only needed for the preferred compiler if not compiler.name in preferred_compiler: @@ -605,7 +643,4 @@ def custom_sort_key(entry): f.write(module_content) logging.info(" ... writing {}".format(mpi_module_file)) - del COMPILER_SUBSTITUTES_SAVE - del MODULEPATHS_SAVE - logging.info("Metamodule generation completed successfully in {}".format(meta_module_dir)) From f6f795a8eb1bb5f6879beb0f0b82f609dc7494ca Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 1 Aug 2025 14:34:16 -0600 Subject: [PATCH 017/163] Incremental steps towards a better setup-meta-modules extension --- .../spack-stack/stack/meta_modules.py | 246 +++---- .../stack/meta_modules.py.backup.20250801 | 687 ++++++++++++++++++ .../spack-stack/stack/templates/compiler | 1 - .../spack-stack/stack/templates/compiler.lua | 1 - .../jcsda-emc/spack-stack/stack/templates/mpi | 1 - .../spack-stack/stack/templates/mpi.lua | 1 - 6 files changed, 784 insertions(+), 153 deletions(-) create mode 100755 spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250801 diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index d0e32fd13..e7077a625 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -39,7 +39,6 @@ SUBSTITUTES_TEMPLATE = { "MODULELOADS": "", - "MODULEPREREQS": "", "MODULEPATHS": "", "CC": "", "CXX": "", @@ -54,6 +53,7 @@ } +# DH* TODO: IMPORT THIS FROM SPACK, THEY MAINTAIN THE SAME DICTIONARY COMPILER_TRANSLATION_TABLE = { "gcc" : "gcc", "clang" : "llvm", @@ -113,27 +113,20 @@ def envmod_command(module_choice, action, env_name, env_values): def module_load_command(module_choice, module): if module_choice == "lmod": - return 'load("{}")\n'.format(module) + return f"""load("{module}") +prereq("{module}")\n""" else: - return """if {{ [ module-info mode load ] && ![ is-loaded {0} ] }} {{ - module load {0} -}}\n""".format( - module - ) - - -def module_prereq_command(module_choice, module): - if module_choice == "lmod": - return 'prereq("{}")\n'.format(module) - else: - return "" + return f"""if {{ [ module-info mode load ] && ![ is-loaded {module} ] }} {{ + module load {module} +}}\n""" def modulepath_prepend_command(module_choice, modulepath): - if module_choice == "lmod": - return 'prepend_path("MODULEPATH", "{}")\n'.format(modulepath) - else: - return "prepend-path {{MODULEPATH}} {{{}}}\n".format(modulepath) + return envmod_command(module_choice, "prepend_path", "MODULEPATH", modulepath) + #if module_choice == "lmod": + # return 'prepend_path("MODULEPATH", "{}")\n'.format(modulepath) + #else: + # return "prepend-path {{MODULEPATH}} {{{}}}\n".format(modulepath) def substitute_config_vars(config_str): @@ -170,6 +163,49 @@ def substitute_config_vars(config_str): return config_str +def get_preferred_compiler(): + """DH* 20250730 - DO WE NEED BOTH WAYS TO DETERMINE THE PREFERRED COMPILER? + # MAYBE KEEP FOR NOW UNTIL WE KNOW BETTER WHAT IS NEEDED AND WHAT NOT. TODO: + UPDATE DOCSTRING""" + # Method 1 + try: + preferred_compilers = spack.config.get("packages")["all"]["require"] + except: + raise Exception( + """Unable to detect preferred compiler from environment. + Does the environment have the config entry 'packages:all:require?'""" + ) + if len(preferred_compilers)>1: + raise Exception(f"Invalid value for packages:all:require is {preferred_compilers}") + match = re.search(r'%\[when=%fortran\](\S+)', preferred_compilers[0]) + if match and match.group(1) in COMPILER_TRANSLATION_TABLE.keys(): + preferred_compiler_v1 = COMPILER_TRANSLATION_TABLE[match.group(1)] + else: + raise Exception(f"Invalid value for packages:all:require {preferred_compilers[0]}") + # Method 2 + try: + preferred_compilers = spack.config.get("packages")["fortran"]["require"] + except: + raise Exception( + """Unable to detect preferred compiler from environment. + Does the environment have the config entry 'packages:fortran:require?'""" + ) + if len(preferred_compilers)>1: + raise Exception(f"Invalid value for packages:fortran:require is {preferred_compilers}") + if preferred_compilers[0] in COMPILER_TRANSLATION_TABLE.values(): + preferred_compiler_v2 = preferred_compilers[0] + else: + raise Exception(f"Invalid value for packages:fortran:require {preferred_compilers[0]}") + if preferred_compiler_v1 == preferred_compiler_v2: + preferred_compiler = preferred_compiler_v1 + del preferred_compilers + del preferred_compiler_v1 + del preferred_compiler_v2 + else: + raise Exception(f"Multiple preferred compilers in spack config: {preferred_compiler_v1} vs {preferred_compiler_v2}") + return preferred_compiler + + def remove_compiler_prefices_from_tcl_modulefiles(modulepath, compiler_list, mpi_provider): """Remove compiler and mpi prefices from tcl modulefiles in modulepath""" logging.info(f" ... ... removing compiler/mpi prefices from tcl modulefiles in {modulepath}") @@ -183,11 +219,9 @@ def remove_compiler_prefices_from_tcl_modulefiles(modulepath, compiler_list, mpi for ffile in files: filepath = os.path.join(root, ffile) logging.debug(f" ... ... ... {filepath}") - # First, compiler-only dependent modules - # These can depend on other compilers than the - # compiler this MPI was built with, loop over all for pattern in module_replace_patterns: for compiler in compiler_list: + # First, compiler-dependent modules (compiler_name, compiler_version) = compiler.split("@") cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( pattern, compiler_name, compiler_version, filepath, sed_syntax_fix @@ -243,15 +277,6 @@ def setup_meta_modules(): module_choice = module_config["default"]["enable"][0] logging.info(" ... configured to use {} modules".format(module_choice)) - ## Need to set a few variables when tcl modules are used - #if module_choice == "tcl": - # module_replace_patterns = ["is-loaded", "module load", "depends-on"] - # # sed syntax differs on macOS - # if sys.platform == "darwin": - # sed_syntax_fix = "''" - # else: - # sed_syntax_fix = "" - # Top-level module directory module_dir = substitute_config_vars(module_config["default"]["roots"][module_choice]) if not os.path.isabs(module_dir): @@ -260,7 +285,7 @@ def setup_meta_modules(): module_dir = os.path.realpath(module_dir) logging.info(f" ... module directory: {module_dir}") - # Get all specs and determine compilers and mpi providers + # Get all specs and determine compilers hashes = env.all_hashes() specs = spack.store.STORE.db.query(hashes=hashes) q = ProviderIndex(specs=specs, repository=spack.repo.PATH) @@ -274,40 +299,24 @@ def setup_meta_modules(): logging.info(f" ... compilers: {compilers}") # To remove compiler prefices from tcl modulefiles, we need - # a mock compiler "none@none" for external packages. Also - # need a list for lmod to check core compiler - #if module_choice == "tcl": - compiler_list = [x.name+"@"+str(x.version) for x in compilers] + ["none@none"] + # a mock compiler "none@none" for external packages. We also + # need this a list for lmod to check the core compiler, but we + # don't need the mock compiler. + if module_choice == "lmod": + compiler_list = [x.name+"@"+str(x.version) for x in compilers] + else: + compiler_list = [x.name+"@"+str(x.version) for x in compilers] + ["none@none"] logging.debug(f" ... compiler_list: {compiler_list}") - mpi_providers = q.providers_for("mpi") - if not len(mpi_providers)==1: - raise Exception(f"Expected exactly one MPI provider, but got {mpi_providers}") - logging.info(f" ... mpi_providers: {mpi_providers}") - # Core compilers is only a valid options for lmod if module_choice == "lmod": # Determine core compiler(s) and make sure they are not used - core_compilers = module_config["default"][module_choice]["core_compilers"] + core_core_compilers = module_config["default"][module_choice]["core_compilers"] logging.info(" ... core compilers: {}".format(core_compilers)) # Check that none of the compilers used for the stack is a core compiler for core_compiler in core_compilers: if any(core_compiler in x for x in compiler_list): - raise Exception( - """Not supported: compiler used for environment - is in list of core compilers""" - ) - - # Prepare meta module directory - logging.info("Preparing meta module directory ...") - meta_module_dir = os.path.join(module_dir, "Core") - if not os.path.isdir(meta_module_dir): - os.mkdir(meta_module_dir) - logging.info(" ... meta module directory : {}".format(meta_module_dir)) - - - # DH* 20250730 - DO WE NEED BOTH WAYS TO DETERMINE THE PREFERRED COMPILER? - # MAYBE KEEP FOR NOW UNTIL WE KNOW BETTER WHAT IS NEEDED AND WHAT NOT. + raise Exception("Not supported: spack-stack compilers in core compilers") # Determine the preferred compiler and sort the list of compilers so that # the preferred compiler comes last. This is so that all other compilers @@ -315,43 +324,8 @@ def setup_meta_modules(): # takes it and adds it to the stack-COMPILER metamodule. Likewise, we need # to save the list of compiler substitutions from the preferred compiler # so that we have access to it when we build the MPI meta module. - try: - preferred_compilers = spack.config.get("packages")["all"]["require"] - except: - raise Exception( - """Unable to detect preferred compiler from environment. - Does the environment have the config entry 'packages:all:require?'""" - ) - if len(preferred_compilers)>1: - raise Exception(f"Invalid value for packages:all:require is {preferred_compilers}") - match = re.search(r'%\[when=%fortran\](\S+)', preferred_compilers[0]) - if match and match.group(1) in COMPILER_TRANSLATION_TABLE.keys(): - preferred_compiler_v1 = COMPILER_TRANSLATION_TABLE[match.group(1)] - else: - raise Exception(f"Invalid value for packages:all:require {preferred_compilers[0]}") - - try: - preferred_compilers = spack.config.get("packages")["fortran"]["require"] - except: - raise Exception( - """Unable to detect preferred compiler from environment. - Does the environment have the config entry 'packages:fortran:require?'""" - ) - if len(preferred_compilers)>1: - raise Exception(f"Invalid value for packages:fortran:require is {preferred_compilers}") - if preferred_compilers[0] in COMPILER_TRANSLATION_TABLE.values(): - preferred_compiler_v2 = preferred_compilers[0] - else: - raise Exception(f"Invalid value for packages:fortran:require {preferred_compilers[0]}") - if preferred_compiler_v1 == preferred_compiler_v2: - preferred_compiler = preferred_compiler_v1 - del preferred_compilers - del preferred_compiler_v1 - del preferred_compiler_v2 - else: - raise Exception(f"Multiple preferred compilers in spack config: {preferred_compiler_v1} vs {preferred_compiler_v2}") + preferred_compiler = get_preferred_compiler() logging.info(" ... preferred compiler: {}".format(preferred_compiler)) - # *DH 20250730 # Sort the list using a custom key def custom_sort_key(entry): @@ -360,6 +334,19 @@ def custom_sort_key(entry): return (1 if preferred_compiler in entry else 0, entry) compilers = sorted(compilers, key=custom_sort_key) + # Get mpi providers (currently only one mpi provider is supported) + mpi_providers = q.providers_for("mpi") + if not len(mpi_providers)==1: + raise Exception(f"Expected exactly one MPI provider, but got {mpi_providers}") + logging.info(f" ... mpi_providers: {mpi_providers}") + + # Prepare meta module directory + logging.info("Preparing meta module directory ...") + meta_module_dir = os.path.join(module_dir, "Core") + if not os.path.isdir(meta_module_dir): + os.mkdir(meta_module_dir) + logging.info(" ... meta module directory : {}".format(meta_module_dir)) + # Create compiler modules logging.info("Creating compiler modules ...") @@ -369,14 +356,14 @@ def custom_sort_key(entry): # Collect and save modulepaths for the preferred compiler MODULEPATHS_SAVE = [] - # Append modulepath for external specs and for specs without compiler dependencies - modulepath_save = os.path.join(module_dir, "none", "none") - if not os.path.isdir(modulepath_save): - os.makedirs(modulepath_save) - logging.info(" ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) - MODULEPATHS_SAVE.append(modulepath_save) - # For tcl modules remove the compiler prefices from the module contents + # For tcl, append modulepath for external specs and for specs without + # compiler dependencies; remove the compiler prefices from the moduless if module_choice == "tcl": + modulepath_save = os.path.join(module_dir, "none", "none") + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + logging.info(" ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) + MODULEPATHS_SAVE.append(modulepath_save) remove_compiler_prefices_from_tcl_modulefiles( modulepath_save, compiler_list, @@ -420,15 +407,11 @@ def custom_sort_key(entry): if compiler.external and compiler.external_modules: for module in compiler.external_modules: substitutes["MODULELOADS"] += module_load_command(module_choice, module) - substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) else: module = "{}/{}".format(compiler.name, compiler.version) substitutes["MODULELOADS"] += module_load_command(module_choice, module) - substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") - substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) - logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) # Compiler environment variables; names are lowercase in spack substitutes["CC"] = compiler.extra_attributes["compilers"]["c"] @@ -450,7 +433,7 @@ def custom_sort_key(entry): substitutes["COMPFLAGS"] = substitutes["COMPFLAGS"].rstrip("\n") logging.debug(" ... ... COMPFLAGS: {}".format(substitutes["COMPFLAGS"])) - # Environment variables; case-sensitive in spack + # Environment variables if "environment" in compiler.extra_attributes.keys(): for action in compiler.extra_attributes["environment"].keys(): for env_name in compiler.extra_attributes["environment"][action]: @@ -461,36 +444,6 @@ def custom_sort_key(entry): env_name, env_values ) - # DH* CAN WE SIMPLIFY THIS WITH A LOOP AND A GENERIC FUNCTION? - # append_path - if "append_path" in compiler.extra_attributes["environment"].keys(): - for env_name in compiler.extra_attributes["environment"]["append_path"]: - env_values = compiler.extra_attributes["environment"]["append_path"][env_name] - substitutes["ENVVARS"] += append_path_command( - module_choice, env_name, env_values - ) - # prepend_path - if "prepend_path" in compiler.extra_attributes["environment"].keys(): - for env_name in compiler.extra_attributes["environment"]["prepend_path"]: - env_values = compiler.extra_attributes["environment"]["prepend_path"][env_name] - substitutes["ENVVARS"] += prepend_path_command( - module_choice, env_name, env_values - ) - # set - if "set" in compiler.extra_attributes["environment"].keys(): - for env_name in compiler.extra_attributes["environment"]["set"]: - env_values = compiler.extra_attributes["environment"]["set"][env_name] - substitutes["ENVVARS"] += setenv_command( - module_choice, env_name, env_values - ) - # unset - if "unset" in compiler.extra_attributes["environment"].keys(): - for env_name in compiler.extra_attributes["environment"]["unset"]: - env_values = compiler.extra_attributes["environment"]["unset"][env_name] - substitutes["ENVVARS"] += unsetenv_command( - module_choice, env_name, env_values - ) - # *DH substitutes["ENVVARS"] = substitutes["ENVVARS"].rstrip("\n") logging.debug(" ... ... ENVVARS : {}".format(substitutes["ENVVARS"])) @@ -514,27 +467,26 @@ def custom_sort_key(entry): with open(compiler_module_file, "w") as f: f.write(module_content) logging.info(" ... writing {}".format(compiler_module_file)) + # If this is the last compiler in the list (i.e. the preferred compiler), - # then save the substitutes for later use for building the MPI meta module. + # then save the substitutes for later when building the MPI meta module if compiler == compilers[-1]: COMPILER_SUBSTITUTES_SAVE = substitutes # Collect and save modulepaths for MPI with preferred compiler MODULEPATHS_SAVE = [] - # Create mpi modules - as of July 2025, only one - # MPI provider is allowed (for the preferred compiler) + # Create mpi modules - currently, only one mpi provider is allowed for mpi_provider in mpi_providers: - # Append modulepath for external specs and specs without compiler dependencies - modulepath_save = os.path.join(module_dir, mpi_provider.name, str(mpi_provider.version), "none", "none") - if not os.path.isdir(modulepath_save): - os.makedirs(modulepath_save) - logging.info(" ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) - MODULEPATHS_SAVE.append(modulepath_save) - - # For tcl modules remove the compiler/mpi prefices from the module contents + # For tcl, append modulepath for external specs and for specs without + # compiler dependencies; remove the compiler/mpi prefices from the moduless if module_choice == "tcl": + modulepath_save = os.path.join(module_dir, mpi_provider.name, str(mpi_provider.version), "none", "none") + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + logging.info(" ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) + MODULEPATHS_SAVE.append(modulepath_save) remove_compiler_prefices_from_tcl_modulefiles( modulepath_save, compiler_list, @@ -582,15 +534,11 @@ def custom_sort_key(entry): if mpi_provider.external and mpi_provider.external_modules: for module in mpi_provider.external_modules: substitutes["MODULELOADS"] += module_load_command(module_choice, module) - substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) else: module = "{}/{}".format(mpi_provider.name, mpi_provider.version) substitutes["MODULELOADS"] += module_load_command(module_choice, module) - substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") - substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) - logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) # Compiler wrapper environment variables if mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers": diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250801 b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250801 new file mode 100755 index 000000000..9ec04f110 --- /dev/null +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250801 @@ -0,0 +1,687 @@ +#!/usr/bin/env python3 + +import copy +import logging +import os +import re +import sys + +import spack +import spack.environment as ev +# import spack.repo +from spack.provider_index import ProviderIndex + +# logging.basicConfig(level=logging.INFO) +logging.basicConfig(format="%(message)s", level=logging.DEBUG) + +# Get basic directory information +logging.info("Configuring basic directory information ...") +this_script_dir = os.path.realpath(os.path.split(__file__)[0]) +base_dir = os.path.realpath(os.path.join(this_script_dir, "..")) +spack_dir = spack.paths.spack_root +logging.info(" ... script directory: {}".format(this_script_dir)) +logging.info(" ... base directory: {}".format(base_dir)) +logging.info(" ... spack directory: {}".format(spack_dir)) + +# Templates for creating compiler modules +COMPILER_TEMPLATES = { + "lmod": os.path.join(this_script_dir, "templates/compiler.lua"), + "tcl": os.path.join(this_script_dir, "templates/compiler"), +} +MPI_TEMPLATES = { + "lmod": os.path.join(this_script_dir, "templates/mpi.lua"), + "tcl": os.path.join(this_script_dir, "templates/mpi"), +} +MODULE_FILE_EXTENSION = { + "lmod": ".lua", + "tcl": "" +} + +SUBSTITUTES_TEMPLATE = { + "MODULELOADS": "", + "MODULEPREREQS": "", + "MODULEPATHS": "", + "CC": "", + "CXX": "", + "F77": "", + "FC": "", + "COMPFLAGS": "", + "ENVVARS": "", + "MPICC": "", + "MPICXX": "", + "MPIF77": "", + "MPIROOT": "", +} + + +COMPILER_TRANSLATION_TABLE = { + "gcc" : "gcc", + "clang" : "llvm", + "intel" : "intel-oneapi-compilers-classic", + "oneapi" : "intel-oneapi-compilers", +} + + +def setenv_command(module_choice, key, value): + if module_choice == "lmod": + return 'setenv("{}", "{}")\n'.format(key, value) + else: + return "setenv {{{}}} {{{}}}\n".format(key, value) + + +def unsetenv_command(module_choice, key, value): + if module_choice == "lmod": + return 'unsetenv("{}", "{}")\n'.format(key, value) + else: + return "unsetenv {{{}}} {{{}}}\n".format(key, value) + + +def append_path_command(module_choice, key, value): + if module_choice == "lmod": + return 'append_path("{}", "{}")\n'.format(key, value) + else: + return "append-path {{{}}} {{{}}}\n".format(key, value) + + +def prepend_path_command(module_choice, key, value): + if module_choice == "lmod": + return 'prepend_path("{}", "{}")\n'.format(key, value) + else: + return "prepend-path {{{}}} {{{}}}\n".format(key, value) + + +def module_load_command(module_choice, module): + if module_choice == "lmod": + return 'load("{}")\n'.format(module) + else: + return """if {{ [ module-info mode load ] && ![ is-loaded {0} ] }} {{ + module load {0} +}}\n""".format( + module + ) + + +def module_prereq_command(module_choice, module): + if module_choice == "lmod": + return 'prereq("{}")\n'.format(module) + else: + return "" + + +def modulepath_prepend_command(module_choice, modulepath): + if module_choice == "lmod": + return 'prepend_path("MODULEPATH", "{}")\n'.format(modulepath) + else: + return "prepend-path {{MODULEPATH}} {{{}}}\n".format(modulepath) + + +def substitute_config_vars(config_str): + """ + Substitute spack-specific and environment variables that may be present + in configuration files. See: + https://spack.readthedocs.io/en/latest/configuration.html#config-file-variables + """ + spack_vars = { + "ENV": ev.active_environment().path, + "SPACK": os.getenv("SPACK_ROOT"), + "TEMPDIR": None, + "USER": os.getenv("HOME"), + "USER_CACHE_PATH": os.path.join(os.getenv("HOME"), ".spack"), + } + + if config_str.startswith("~"): + config_str = config_str.replace("~", os.getenv("HOME")) + + # Get var as it appears in the string (e.g. ${env}), and its name (e.g. env) + matches = re.findall(r"(\$(\w+))|(\${(\w+)})", config_str) + for match in matches: + if match[0]: + pair = (match[0], match[1]) + else: + pair = (match[2], match[3]) + + var_string = pair[0] + var_name = pair[1].upper() + + sub_value = spack_vars[var_name] if spack_vars[var_name] else os.getenv(var_name) + config_str = config_str.replace(var_string, sub_value) + + return config_str + + +def setup_meta_modules(): + # Find currently active spack environment, activate here + logging.info("Configuring active spack environment ...") + env_dir = ev.active_environment().path + if not env_dir: + raise Exception("No active spack environment") + env = spack.environment.Environment(env_dir) + spack.environment.environment.activate(env) + logging.info(" ... environment directory: {}".format(env_dir)) + + # Parse spack main config from environment + logging.info("Parsing spack environment main config ...") + main_config = spack.config.get("config") + install_dir = substitute_config_vars(main_config["install_tree"]["root"]) + + if not os.path.isabs(install_dir): + install_dir = os.path.realpath(os.path.join(env_dir, install_dir)) + else: + install_dir = os.path.realpath(install_dir) + logging.info(" ... install directory: {}".format(install_dir)) + + # Parse spack module config from environment + logging.info("Parsing spack environment modules config ...") + module_config = spack.config.get("modules") + + # Check which modules are used - tcl or lmod (can only be one) + if len(module_config["default"]["enable"]) > 1: + raise Exception("Can use either lmod or tcl modules, not both") + module_choice = module_config["default"]["enable"][0] + logging.info(" ... configured to use {} modules".format(module_choice)) + + # Need to set a few variables when tcl modules are used + if module_choice == "tcl": + module_replace_patterns = ["is-loaded", "module load", "depends-on"] + # sed syntax differs on macOS + if sys.platform == "darwin": + sed_syntax_fix = "''" + else: + sed_syntax_fix = "" + + # Top-level module directory + module_dir = substitute_config_vars(module_config["default"]["roots"][module_choice]) + if not os.path.isabs(module_dir): + module_dir = os.path.realpath(os.path.join(env_dir, module_dir)) + else: + module_dir = os.path.realpath(module_dir) + logging.info(f" ... module directory: {module_dir}") + + # Get all specs and determine compilers and mpi providers + hashes = env.all_hashes() + specs = spack.store.STORE.db.query(hashes=hashes) + q = ProviderIndex(specs=specs, repository=spack.repo.PATH) + + c_providers = q.providers_for("c") + cxx_providers = q.providers_for("cxx") + fortran_providers = q.providers_for("fortran") + compilers = list(set(c_providers + cxx_providers + fortran_providers)) + if not compilers: + raise Exception("No compilers found") + logging.info(f" ... compilers: {compilers}") + + # To remove compiler prefices from tcl modulefiles, we need + # a mock compiler "none@none" for external packages. Also + # need a list for lmod to check core compiler + #if module_choice == "tcl": + mock_compiler_list = [x.name+"@"+str(x.version) for x in compilers] + ["none@none"] + logging.debug(f" ... mock_compiler_list: {mock_compiler_list}") + + mpi_providers = q.providers_for("mpi") + if not len(mpi_providers)==1: + raise Exception(f"Expected exactly one MPI provider, but got {mpi_providers}") + logging.info(f" ... mpi_providers: {mpi_providers}") + + # Core compilers is only a valid options for lmod + if module_choice == "lmod": + # Determine core compiler(s) and make sure they are not used + core_compilers = module_config["default"][module_choice]["core_compilers"] + logging.info(" ... core compilers: {}".format(core_compilers)) + # Check that none of the compilers used for the stack is a core compiler + for core_compiler in core_compilers: + if any(core_compiler in x for x in mock_compiler_list): + raise Exception( + """Not supported: compiler used for environment + is in list of core compilers""" + ) + + # Prepare meta module directory + logging.info("Preparing meta module directory ...") + meta_module_dir = os.path.join(module_dir, "Core") + if not os.path.isdir(meta_module_dir): + os.mkdir(meta_module_dir) + logging.info(" ... meta module directory : {}".format(meta_module_dir)) + + + # DH* 20250730 - DO WE NEED BOTH WAYS TO DETERMINE THE PREFERRED COMPILER? + # MAYBE KEEP FOR NOW UNTIL WE KNOW BETTER WHAT IS NEEDED AND WHAT NOT. + + # Determine the preferred compiler and sort the list of compilers so that + # the preferred compiler comes last. This is so that all other compilers + # populate the MODULEPATHS_SAVE list before the preferred compiler + # takes it and adds it to the stack-COMPILER metamodule. Likewise, we need + # to save the list of compiler substitutions from the preferred compiler + # so that we have access to it when we build the MPI meta module. + try: + preferred_compilers = spack.config.get("packages")["all"]["require"] + except: + raise Exception( + """Unable to detect preferred compiler from environment. + Does the environment have the config entry 'packages:all:require?'""" + ) + if len(preferred_compilers)>1: + raise Exception(f"Invalid value for packages:all:require is {preferred_compilers}") + match = re.search(r'%\[when=%fortran\](\S+)', preferred_compilers[0]) + if match and match.group(1) in COMPILER_TRANSLATION_TABLE.keys(): + preferred_compiler_v1 = COMPILER_TRANSLATION_TABLE[match.group(1)] + else: + raise Exception(f"Invalid value for packages:all:require {preferred_compilers[0]}") + + try: + preferred_compilers = spack.config.get("packages")["fortran"]["require"] + except: + raise Exception( + """Unable to detect preferred compiler from environment. + Does the environment have the config entry 'packages:fortran:require?'""" + ) + if len(preferred_compilers)>1: + raise Exception(f"Invalid value for packages:fortran:require is {preferred_compilers}") + if preferred_compilers[0] in COMPILER_TRANSLATION_TABLE.values(): + preferred_compiler_v2 = preferred_compilers[0] + else: + raise Exception(f"Invalid value for packages:fortran:require {preferred_compilers[0]}") + if preferred_compiler_v1 == preferred_compiler_v2: + preferred_compiler = preferred_compiler_v1 + del preferred_compilers + del preferred_compiler_v1 + del preferred_compiler_v2 + else: + raise Exception(f"Multiple preferred compilers in spack config: {preferred_compiler_v1} vs {preferred_compiler_v2}") + logging.info(" ... preferred compiler: {}".format(preferred_compiler)) + # *DH 20250730 + + # Sort the list using a custom key + def custom_sort_key(entry): + # Return a tuple where the first element is 1 if the entry contains the word, else 0 + # The second element is the entry itself for natural sorting within groups + return (1 if preferred_compiler in entry else 0, entry) + compilers = sorted(compilers, key=custom_sort_key) + + # Create compiler modules + logging.info("Creating compiler modules ...") + + # Collect and save modulepaths for the preferred compiler + MODULEPATHS_SAVE = [] + + # Append modulepath for external specs and specs without compiler dependencies + modulepath_save = os.path.join(module_dir, "none", "none") + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + logging.info(" ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) + MODULEPATHS_SAVE.append(modulepath_save) + # For tcl modules remove the compiler prefices from the module contents + if module_choice == "tcl": + logging.info( + " ... ... removing compiler prefices from tcl modulefiles in {}".format( + modulepath_save + ) + ) + + for root, ddir, files in os.walk(modulepath_save): + for ffile in files: + filepath = os.path.join(root, ffile) + logging.debug( + " ... ... ... removing compiler prefices in {}".format(filepath) + ) + for pattern in module_replace_patterns: + for tmp_compiler in mock_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") + cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( + pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + + # Initialize saved substitutes to None (populate for preferred compiler later) + COMPILER_SUBSTITUTES_SAVE = None + + for compiler in compilers: + logging.info(f" ... configuring compiler {compiler.name}@{compiler.version}") + + modulepath_save = os.path.join(module_dir, compiler.name, str(compiler.version)) + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + logging.info(" ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) + MODULEPATHS_SAVE.append(modulepath_save) + + # For tcl modules remove the compiler prefices from the module contents + if module_choice == "tcl": + logging.info( + " ... ... removing compiler prefices from tcl modulefiles in {}".format( + modulepath_save + ) + ) + + for root, ddir, files in os.walk(modulepath_save): + for ffile in files: + filepath = os.path.join(root, ffile) + logging.debug( + " ... ... ... removing compiler prefices in {}".format(filepath) + ) + for pattern in module_replace_patterns: + for tmp_compiler in mock_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") + cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( + pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + + # The remainder of the loop is only needed for the preferred compiler + if not compiler.name in preferred_compiler: + continue + + logging.info( + " ... configuring stack compiler {}@{}".format(compiler.name, compiler.version) + ) + compiler_module_dir = os.path.join(meta_module_dir, "stack-" + compiler.name) + compiler_module_file = os.path.join( + compiler_module_dir, str(compiler.version) + MODULE_FILE_EXTENSION[module_choice] + ) + substitutes = SUBSTITUTES_TEMPLATE.copy() + + if not compiler.external: + raise Exception(f"spack-built compiler {compiler} not yet supported") + + # Use existing modules for external mpi providers; otherwise, use spack-built module + if compiler.external and compiler.external_modules: + for module in compiler.external_modules: + substitutes["MODULELOADS"] += module_load_command(module_choice, module) + substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) + else: + module = "{}/{}".format(compiler.name, compiler.version) + substitutes["MODULELOADS"] += module_load_command(module_choice, module) + substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) + substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") + substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") + logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) + logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) + + # Compiler environment variables; names are lowercase in spack + substitutes["CC"] = compiler.extra_attributes["compilers"]["c"] + substitutes["CXX"] = compiler.extra_attributes["compilers"]["cxx"] + substitutes["F77"] = compiler.extra_attributes["compilers"]["fortran"] + substitutes["FC"] = compiler.extra_attributes["compilers"]["fortran"] + logging.debug(" ... ... CC : {}".format(substitutes["CC"])) + logging.debug(" ... ... CXX : {}".format(substitutes["CXX"])) + logging.debug(" ... ... F77 : {}".format(substitutes["F77"])) + logging.debug(" ... ... FC' : {}".format(substitutes["FC"])) + + # Compiler flags; names are lowercase in spack + if "flags" in compiler.extra_attributes.keys(): + for flag_name in compiler.extra_attributes["flags"].keys(): + flag_values = compiler["compiler"]["flags"][flag_name] + substitutes["COMPFLAGS"] += setenv_command( + module_choice, flag_name.upper(), flag_values + ) + substitutes["COMPFLAGS"] = substitutes["COMPFLAGS"].rstrip("\n") + logging.debug(" ... ... COMPFLAGS: {}".format(substitutes["COMPFLAGS"])) + + # Environment variables; case-sensitive in spack + if "environment" in compiler.extra_attributes.keys(): + # DH* CAN WE SIMPLIFY THIS WITH A LOOP AND A GENERIC FUNCTION? + # append_path + if "append_path" in compiler.extra_attributes["environment"].keys(): + for env_name in compiler.extra_attributes["environment"]["append_path"]: + env_values = compiler.extra_attributes["environment"]["append_path"][env_name] + substitutes["ENVVARS"] += append_path_command( + module_choice, env_name, env_values + ) + # prepend_path + if "prepend_path" in compiler.extra_attributes["environment"].keys(): + for env_name in compiler.extra_attributes["environment"]["prepend_path"]: + env_values = compiler.extra_attributes["environment"]["prepend_path"][env_name] + substitutes["ENVVARS"] += prepend_path_command( + module_choice, env_name, env_values + ) + # set + if "set" in compiler.extra_attributes["environment"].keys(): + for env_name in compiler.extra_attributes["environment"]["set"]: + env_values = compiler.extra_attributes["environment"]["set"][env_name] + substitutes["ENVVARS"] += setenv_command( + module_choice, env_name, env_values + ) + # unset + if "unset" in compiler.extra_attributes["environment"].keys(): + for env_name in compiler.extra_attributes["environment"]["unset"]: + env_values = compiler.extra_attributes["environment"]["unset"][env_name] + substitutes["ENVVARS"] += unsetenv_command( + module_choice, env_name, env_values + ) + # *DH + substitutes["ENVVARS"] = substitutes["ENVVARS"].rstrip("\n") + logging.debug(" ... ... ENVVARS : {}".format(substitutes["ENVVARS"])) + + # Spack compiler module hierarchy - append all saved modulepaths + for modulepath in MODULEPATHS_SAVE: + substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) + substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") + logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) + + # Read compiler template into module_content string + with open(COMPILER_TEMPLATES[module_choice]) as f: + module_content = f.read() + + # Substitute variables in module_content + for key in substitutes.keys(): + module_content = module_content.replace("@{}@".format(key), substitutes[key]) + + # Write compiler lua module + if not os.path.isdir(compiler_module_dir): + os.makedirs(compiler_module_dir) + with open(compiler_module_file, "w") as f: + f.write(module_content) + logging.info(" ... writing {}".format(compiler_module_file)) + # If this is the last compiler in the list (i.e. the preferred compiler), + # then save the substitutes for later use for building the MPI meta module. + if compiler == compilers[-1]: + COMPILER_SUBSTITUTES_SAVE = substitutes + + del MODULEPATHS_SAVE + + # Collect and save modulepaths for MPI with preferred compiler + MODULEPATHS_SAVE = [] + + # Create mpi modules - as of July 2025, only one + # MPI provider is allowed (for the preferred compiler) + for mpi_provider in mpi_providers: + + # Append modulepath for external specs and specs without compiler dependencies + modulepath_save = os.path.join(module_dir, mpi_provider.name, str(mpi_provider.version), "none", "none") + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + logging.info(" ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) + MODULEPATHS_SAVE.append(modulepath_save) + + # For tcl modules remove the compiler/mpi prefices from the module contents + if module_choice == "tcl": + logging.info( + " ... ... removing no-compiler/mpi prefices from tcl modulefiles in {}".format( + modulepath_save + ) + ) + for root, ddir, files in os.walk(modulepath_save): + for ffile in files: + filepath = os.path.join(root, ffile) + logging.debug( + " ... ... ... removing no-compiler/mpi prefices in {}".format( + filepath + ) + ) + # First, compiler-only dependent modules + # These can depend on other compilers than the + # compiler this MPI was built with, loop over all + for pattern in module_replace_patterns: + for tmp_compiler in mock_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") + cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( + pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + # Then, compiler+mpi-dependent modules + # By definition, these can only depend on the compiler that + # this MPI was built with - no need to loop over all compilers + for pattern in module_replace_patterns: + for tmp_compiler in mock_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") + cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format( + pattern, + mpi_provider.name, + mpi_provider.version, + tmp_compiler_name, + tmp_compiler_version, + filepath, + sed_syntax_fix, + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + + for compiler in compilers: + logging.info( + " ... configuring stack mpi library {}@{} for compiler {}@{}".format( + mpi_provider.name, mpi_provider.version, compiler.name, compiler.version + ) + ) + + # Spack mpi+compiler module hierarchy + modulepath_save = os.path.join( + module_dir, mpi_provider.name, str(mpi_provider.version), compiler.name, str(compiler.version) + ) + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + logging.info(" ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) + MODULEPATHS_SAVE.append(modulepath_save) + + # For tcl modules remove the compiler/mpi prefices from the module contents + if module_choice == "tcl": + logging.info( + " ... ... removing compiler/mpi prefices from tcl modulefiles in {}".format( + modulepath_save + ) + ) + for root, ddir, files in os.walk(modulepath_save): + for ffile in files: + filepath = os.path.join(root, ffile) + logging.debug( + " ... ... ... removing compiler/mpi prefices in {}".format( + filepath + ) + ) + # First, compiler-only dependent modules + # These can depend on other compilers than the + # compiler this MPI was built with, loop over all + for pattern in module_replace_patterns: + for tmp_compiler in mock_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") + cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( + pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + # Then, compiler+mpi-dependent modules + # By definition, these can only depend on the compiler that + # this MPI was built with - no need to loop over all compilers + for pattern in module_replace_patterns: + for tmp_compiler in mock_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") + cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format( + pattern, + mpi_provider.name, + mpi_provider.version, + tmp_compiler_name, + tmp_compiler_version, + filepath, + sed_syntax_fix, + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + + # The remainder of the loop is only needed for the preferred compiler + if not compiler.name in preferred_compiler: + continue + + # Path and name for mpi module file + mpi_module_dir = os.path.join( + module_dir, compiler.name, str(compiler.version), "stack-" + mpi_provider.name + ) + mpi_module_file = os.path.join( + mpi_module_dir, str(mpi_provider.version).split("-")[0] + MODULE_FILE_EXTENSION[module_choice] + ) + + substitutes = SUBSTITUTES_TEMPLATE.copy() + # Use existing modules for external mpi providers; otherwise, use spack-built module + if mpi_provider.external and mpi_provider.external_modules: + for module in mpi_provider.external_modules: + substitutes["MODULELOADS"] += module_load_command(module_choice, module) + substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) + else: + module = "{}/{}".format(mpi_provider.name, mpi_provider.version) + substitutes["MODULELOADS"] += module_load_command(module_choice, module) + substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) + substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") + substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") + logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) + logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) + + # Compiler wrapper environment variables + if mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers": + substitutes["MPICC"] = os.path.join("mpiicx") + substitutes["MPICXX"] = os.path.join("mpiicpx") + if "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and not "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: + substitutes["MPIF77"] = os.path.join("mpiifx") + substitutes["MPIF90"] = os.path.join("mpiifx") + elif not "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: + substitutes["MPIF77"] = os.path.join("mpiifort") + substitutes["MPIF90"] = os.path.join("mpiifort") + else: + raise Exception(f"For {mpi_provider.name}, cannot determine MPI wrapper from FC={COMPILER_SUBSTITUTES_SAVE['FC']}") + elif mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers-classic": + substitutes["MPICC"] = os.path.join("mpiicc") + substitutes["MPICXX"] = os.path.join("mpiicpc") + substitutes["MPIF77"] = os.path.join("mpiifort") + substitutes["MPIF90"] = os.path.join("mpiifort") + else: + substitutes["MPICC"] = os.path.join("mpicc") + substitutes["MPICXX"] = os.path.join("mpic++") + substitutes["MPIF77"] = os.path.join("mpif77") + substitutes["MPIF90"] = os.path.join("mpif90") + # Also set the direct compiler environment variables + substitutes["CC"] = COMPILER_SUBSTITUTES_SAVE["CC"] + substitutes["CXX"] = COMPILER_SUBSTITUTES_SAVE["CXX"] + substitutes["F77"] = COMPILER_SUBSTITUTES_SAVE["F77"] + substitutes["FC"] = COMPILER_SUBSTITUTES_SAVE["FC"] + + # Spack mpi+compiler module hierarchy - append all saved modulepaths + for modulepath in MODULEPATHS_SAVE: + substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) + substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") + logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) + + # Read compiler lua template into module_content string + with open(MPI_TEMPLATES[module_choice]) as f: + module_content = f.read() + + # Substitute variables in module_content + for key in substitutes.keys(): + module_content = module_content.replace( + "@{}@".format(key), substitutes[key] + ) + + # Write mpi lua module + if not os.path.isdir(mpi_module_dir): + os.makedirs(mpi_module_dir) + with open(mpi_module_file, "w") as f: + f.write(module_content) + logging.info(" ... writing {}".format(mpi_module_file)) + + del COMPILER_SUBSTITUTES_SAVE + del MODULEPATHS_SAVE + + logging.info("Metamodule generation completed successfully in {}".format(meta_module_dir)) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler index 0929436be..0da4db8eb 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler @@ -24,7 +24,6 @@ conflict stack-intel-oneapi-compilers # prerequisite modules @MODULELOADS@ -@MODULEPREREQS@ # compiler environment variables setenv {F77} {@F77@} diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua index 0cc0bc1a2..1ff39dbc6 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua @@ -28,7 +28,6 @@ conflict("stack-intel-oneapi-compilers") -- prerequisite modules @MODULELOADS@ -@MODULEPREREQS@ -- compiler environment variables setenv("F77", "@F77@") diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi index 97a7a07dc..ce0cedcfe 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi @@ -16,7 +16,6 @@ conflict stack-openmpi # prerequisite modules @MODULELOADS@ -@MODULEPREREQS@ # mpi wrapper environment variables setenv {MPICC} {@MPICC@} diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua index 7e4099ed2..ec841f2d0 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua @@ -20,7 +20,6 @@ conflict("stack-openmpi") -- prerequisite modules @MODULELOADS@ -@MODULEPREREQS@ -- mpi wrapper environment variables setenv("MPICC", "@MPICC@") From 37bde5c120ab1f8fe55b79401f98e79008b5c696 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 1 Aug 2025 14:51:26 -0600 Subject: [PATCH 018/163] Clean up configs/common/packages*.yaml --- configs/common/packages.yaml | 803 ++++++++++++++-------------- configs/common/packages_gcc.yaml | 50 +- configs/common/packages_oneapi.yaml | 174 +++--- 3 files changed, 510 insertions(+), 517 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index ec588a488..edc8100bc 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -1,412 +1,393 @@ packages: - # - all: - providers: - gl: [opengl] - glu: [openglu] - jpeg: [libjpeg-turbo] - pkgconfig: [pkg-config] - yacc: [bison] - zlib-api: [zlib] - # - awscli-v2: - require: - - '~examples' - bacio: - require: - - '@2.6.0' - bison: - require: - - '@3.4:' - # 1.85 incompatible with ecflow@5.11.4 - use latest 'good version' - boost: - require: - - '@1.84' - - '~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden' - bufr: - require: - - '@12.1.0' - - '+python' - bufr-query: - require: - - '@0.0.4' - - '+python' - cairo: - require: - - '+pic' - cdo: - require: - - any_of: ['@2.3.0 ~openmp'] - when: '%intel@2021:' - message: '2.3.0 is the last version to use C++17, use with Intel Classic' - - any_of: ['@2.3.0 ~openmp'] - when: '%gcc@:10' - message: 2.3.0 is the last version to use C++17, use with GCC 10 and earlier - - any_of: ['@2.3.0 ~openmp'] - when: '%apple-clang@:14' - message: '2.3.0 is the last version to use C++17, use with Apple Clang 14 and earlier' - - any_of: ['@2.0.5 ~openmp'] - when: '%intel@:2020' - message: '2.0.5 is the last version to use C++14, use with Intel Classic 2020 and earlier' - cmake: - require: - - '@3.29:' - - '+ownlibs' - # Attention - when updating also check the various jcsda-emc-bundles env packages - crtm: - require: - - '+fix' - ecbuild: - require: - - '@3.7.2' - eccodes: - require: - - '@2.33.0' - - '+netcdf +png +tools' - ecflow: - require: - - '@5.11.4' - - '+ui' - eckit: - require: - - '@1.28.3' - - 'linalg=eigen,lapack compression=lz4,bzip2' - ecmwf-atlas: - require: - - '@0.40.0' - - '+fckit +trans +tesselation +fftw' - ectrans: - require: - - '@1.5.0' - eigen: - require: - - '@3.4.0' - # Attention - when updating the version also check the common modules.yaml - # config and update the projections for lmod/tcl. - # Also, check the acorn and derecho site configs which have esmf modifications. - esmf: - require: - - '~xerces ~pnetcdf +shared +external-parallelio +python' - - any_of: ['@=8.6.1 snapshot=none', '@=8.8.0 snapshot=none', '@=8.9.0b12 snapshot=b12'] - - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] - when: "%intel" - message: "Extra ESMF compile options for Intel" - #- any_of: ['+python'] - # when: "%gcc" - #- any_of: ['+python'] - # when: "%intel" - #- any_of: ['+python'] - # when: "%oneapi" - #- any_of: ['~python'] - # when: "%clang" - # message: "Disable python variant for ESMF because mpi4py doesn't build with clang" - # To avoid duplicate packages (concretizer bug?) - expat: - require: - - '~libbsd' - fckit: - require: - - '@0.13.2' - - '+eckit' - fftw: - require: - - '@3.3.10' - fiat: - require: - - '@1.4.1' - flex: - # Pin version to avoid duplicates - require: - - '@2.6.4' - fms: - require: - - '@2024.02' - - 'precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io' - fontconfig: - require: - - '+pic' - freetype: - require: - - '+pic' - g2: - require: - - '@3.5.1' - g2c: - require: - - '@2.1.0' - g2tmpl: - require: - - '@1.13.0' - gcc-runtime: - require: - - '%gcc' - gfsio: - require: - - '@1.4.2' - #git-lfs: - # Assume git-lfs is provided, hard to install - # because of dependencies on go/go-bootstrap. - # Note: Uncommenting this entry will break - # the container builds. - #require: '@2.11.0' - ## https://github.com/spack/spack/issues/49358 - ## https://github.com/spack/spack/pull/44323 - #glib: - # require: - # - '@:2.78.3' - ## To avoid duplicate packages being built - #gmake: - # require: - # - one_of: ['@=4.2.1', '@=4.4.1'] - grib-util: - require: - - '@1.4.0' - gsibec: - require: - - '@1.2.1' - gsi-ncdiag: - require: - - '@1.1.2' - #gsl-lite: - # require: - # - '@0.37.0' - hdf: - require: - - '@4.2.15' - - '+external-xdr ~fortran ~netcdf' - hdf5: - require: - - '@1.14.5' - - '+hl +fortran +mpi +threadsafe ~szip' - # Newer versions of hdf-eos2 require manual downloading, avoid - hdf-eos2: - require: - - '@2.20v1.00' - intel-oneapi-compilers-runtime: - require: - - '%oneapi' - intel-oneapi-compilers-classic-runtime: - require: - - '%intel' - ip: - require: - - '@5.4.0' - - 'precision=4,d,8' - ip2: - require: - - '@1.1.2' - #jasper: - # require: - # - '@2.0.32' - jedi-cmake: - require: - - '@1.4.0' - landsfcutil: - require: - - '@2.4.2' - #libjpeg-turbo: - # require: - # - '@2.1.0' - libpng: - require: - - '@1.6.37' - - '+pic' - # libtirpc +gssapi doesn't build with oneapi@2025 - # https://github.com/JCSDA/spack-stack/issues/1441 - libtirpc: - require: - - '~gssapi' - # Newest version of magics needed for oneapi compilers - magics: - require: - - '@4.15.3:' - mapl: - require: - - '@2.53' - - '~shared ~f2py +pflogger' - met: - require: - - '@12.0.1' - - '+python +grib2' - metplus: - require: - - '@6.0.0' - metis: - require: - - '+int64 +real64' - mpich: - require: - - '~hwloc' - nco: - require: - - '~doc' - # ncview - when adding information here, also check Orion - # and Discover site configs - nemsio: - require: - - '@2.5.4' - nemsiogfs: - require: - - '@2.5.3' - nccmp: - require: - - '@1.9.0.1' - ncio: - require: - - '@1.1.2' - netcdf-c: - require: - - '@4.9.2' - - '+dap +mpi ~parallel-netcdf ~szip +parallel_tests build_system=autotools' - netcdf-cxx4: - require: - - '@4.3.1' - netcdf-fortran: - require: - - '@4.6.1' - - '+parallel_tests' - # ninja - when adding information here, also check Discover site config - odc: - require: - - '@1.6.1' - - '~fortran' - openmpi: - require: - - '~internal-hwloc +two_level_namespace' - openssl: - require: - - '+shared' - p4est: - require: - - '@2.8.7' - parallelio: - require: - - '@2.6.2' - # DH* CAN WE UPDATE? NEED A NEW SHARED-INTEL PATCH? - parallel-netcdf: - require: - - '@1.12.3' - pflogger: - require: - - '+mpi' - pixman: - require: - - '+pic' - prod-util: - require: - - '@2.1.1' - proj: - require: - - '~tiff' - # Turn off crypt variant for Python; this leads to build errors - # with Intel in py-cryptography unless external curl and openssl - # are removed, which itself is problematic. - python: - require: - - '@3.11' - - '~crypt' - py-cartopy: - require: - - '+plotting' - ## Pin py-cython to avoid duplicate packages - #py-cython: - # require: - # - '@3' - ## As long as we need the "delayed" variant for py-dask, - ## we are restricted to version 2021.03.0 (last one that has it) - ## and newer than 2021.01.0 because of these issues: - ## https://github.com/JCSDA/spack-stack/issues/1216 - ## https://github.com/pydata/xarray/issues/8917 - #py-dask: - # require: - # - '@2021.3.0' - ## To avoid duplicate packages - #py-flit-core: - # require: - # - '@3.8.0' - py-h5py: - require: - - '~mpi' - # https://github.com/JCSDA/spack-stack/issues/1276 - #py-matplotlib: - # require: - # - '@3.7.4' - # https://github.com/Unidata/netcdf4-python/issues/1389 - # https://github.com/Unidata/netcdf4-python/issues/1389#issuecomment-2599760051 - py-netcdf4: - require: - - '@1.7.2' - - '~mpi' - ## To avoid duplicate packages - #py-numpy: - # require: - # - '@1.26' - ## To avoid duplicate packages - #py-ruamel-yaml: - # require: - # - '@0.17.16' - ## Pin the py-setuptools version to avoid duplicate Python packages - #py-setuptools: - # require: - # - '@69' - ## When making changes here, also check the packages.yaml for atlantis, nautilus, blackpearl - py-torch: - require: - - '+custom-protobuf ~mkldnn' - ## To avoid duplicate packages - #py-urllib3: - # require: - # - '@1.26.12' - ## To avoid duplicate packages - #py-versioneer: - # require: - # - '@0.28' - qt: - require: - - '@5' - scotch: - require: - - '@7.0.4' - - '+mpi+metis~shared~threads~mpi_thread+noarch+esmumps' - sfcio: - require: - - '@1.4.2' - shumlib: - require: - - '@macos_clang_linux_intel_port' - sigio: - require: - - '@2.3.3' - sp: - require: - - '@2.5.0' - - 'precision=4,d,8' - udunits: - require: - - '@2.2.28' - ufs-utils: - require: - - '@1.13.0' - # Note - we can remove upp from stack at some point? - upp: - require: - - '@10.0.10' - # Need extradeps for grib-utils, enable by default to avoid duplicate packages - w3emc: - require: - - '@2.10.0' - - 'precision=4,d,8 +extradeps' - w3nco: - require: - - '@2.4.1' - # See macOS site config for wgrib2 version/variant overrides - wgrib2: - require: - - '@3.6.0' - wrf-io: - require: - - '@1.2.0' - zstd: - require: - - '@1.5.7' - - '+programs' + all: + providers: + jpeg: [libjpeg-turbo] + pkgconfig: [pkg-config] + yacc: [bison] + zlib-api: [zlib] + awscli-v2: + require: + - ~examples + bacio: + require: + - '@2.6.0' + bison: + require: + - '@3.4:' + # 1.85 incompatible with ecflow@5.11.4 - use latest 'good version' + boost: + require: + - '@1.84' + - ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log + - ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization + - ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden + bufr: + require: + - '@12.1.0' + - +python + bufr-query: + require: + - '@0.0.4' + - +python + cairo: + require: + - +pic + cdo: + require: + - any_of: ['@2.3.0 ~openmp'] + when: '%intel@2021:' + message: 2.3.0 is the last version to use C++17, use with Intel Classic + - any_of: ['@2.3.0 ~openmp'] + when: '%gcc@:10' + message: 2.3.0 is the last version to use C++17, use with GCC 10 and earlier + - any_of: ['@2.3.0 ~openmp'] + when: '%apple-clang@:14' + message: 2.3.0 is the last version to use C++17, use with Apple Clang 14 and earlier + - any_of: ['@2.0.5 ~openmp'] + when: '%intel@:2020' + message: 2.0.5 is the last version to use C++14, use with Intel Classic 2020 and earlier + cmake: + require: + - '@3.29:' + - +ownlibs + # Attention - when updating also check the various jcsda-emc-bundles env packages + crtm: + require: + - +fix + ecbuild: + require: + - '@3.7.2' + eccodes: + require: + - '@2.33.0' + - +netcdf +png +tools + ecflow: + require: + - '@5.11.4' + - +ui + eckit: + require: + - '@1.28.3' + - linalg=eigen,lapack compression=lz4,bzip2 + ecmwf-atlas: + require: + - '@0.40.0' + - +fckit +trans +tesselation +fftw + ectrans: + require: + - '@1.5.0' + eigen: + require: + - '@3.4.0' + # Attention - when updating the version also check the common modules.yaml + # config and update the projections for lmod/tcl. + # Also, check the acorn and derecho site configs which have esmf modifications. + esmf: + require: + - ~xerces ~pnetcdf +shared +external-parallelio ~python + - any_of: ['@=8.6.1 snapshot=none', '@=8.8.0 snapshot=none', '@=8.9.0b12 snapshot=b12'] + - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] + when: '%intel' + message: Extra ESMF compile options for Intel + expat: + require: + - ~libbsd + fckit: + require: + - '@0.13.2' + - +eckit + fftw: + require: + - '@3.3.10' + fiat: + require: + - '@1.4.1' + flex: + require: + - '@2.6.4' + fms: + require: + - '@2024.02' + - precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io + fontconfig: + require: + - +pic + freetype: + require: + - +pic + g2: + require: + - '@3.5.1' + g2c: + require: + - '@2.1.0' + g2tmpl: + require: + - '@1.13.0' + gcc-runtime: + require: + - '%gcc' + gfsio: + require: + - '@1.4.2' + ## https://github.com/spack/spack/issues/49358 + ## https://github.com/spack/spack/pull/44323 + #glib: + # require: + # - '@:2.78.3' + ## To avoid duplicate packages being built + #gmake: + # require: + # - one_of: ['@=4.2.1', '@=4.4.1'] + grib-util: + require: + - '@1.4.0' + gsibec: + require: + - '@1.2.1' + gsi-ncdiag: + require: + - '@1.1.2' + #gsl-lite: + # require: + # - '@0.37.0' + hdf: + require: + - '@4.2.15' + - +external-xdr ~fortran ~netcdf + hdf5: + require: + - '@1.14.5' + - +hl +fortran +mpi +threadsafe ~szip + # Newer versions of hdf-eos2 require manual downloading, avoid + hdf-eos2: + require: + - '@2.20v1.00' + intel-oneapi-compilers-runtime: + require: + - '%oneapi' + intel-oneapi-compilers-classic-runtime: + require: + - '%intel' + ip: + require: + - '@5.4.0' + - precision=4,d,8 + ip2: + require: + - '@1.1.2' + #jasper: + # require: + # - '@2.0.32' + jedi-cmake: + require: + - '@1.4.0' + landsfcutil: + require: + - '@2.4.2' + #libjpeg-turbo: + # require: + # - '@2.1.0' + libpng: + require: + - '@1.6.37' + - +pic + # libtirpc +gssapi doesn't build with oneapi@2025 + # https://github.com/JCSDA/spack-stack/issues/1441 + libtirpc: + require: + - ~gssapi + # Newest version of magics needed for oneapi compilers + magics: + require: + - '@4.15.3:' + mapl: + require: + - '@2.53' + - ~shared ~f2py +pflogger + met: + require: + - '@12.0.1' + - +python +grib2 + metplus: + require: + - '@6.0.0' + metis: + require: + - +int64 +real64 + mpich: + require: + - ~hwloc + nco: + require: + - ~doc + # ncview - when adding information here, also check Orion + # and Discover site configs + nemsio: + require: + - '@2.5.4' + nemsiogfs: + require: + - '@2.5.3' + nccmp: + require: + - '@1.9.0.1' + ncio: + require: + - '@1.1.2' + netcdf-c: + require: + - '@4.9.2' + - +dap +mpi ~parallel-netcdf ~szip +parallel_tests build_system=autotools + netcdf-cxx4: + require: + - '@4.3.1' + netcdf-fortran: + require: + - '@4.6.1' + - +parallel_tests + # ninja - when adding information here, also check Discover site config + odc: + require: + - '@1.6.1' + - ~fortran + openmpi: + require: + - ~internal-hwloc +two_level_namespace + openssl: + require: + - +shared + p4est: + require: + - '@2.8.7' + parallelio: + require: + - '@2.6.2' + # DH* CAN WE UPDATE? NEED A NEW SHARED-INTEL PATCH? + parallel-netcdf: + require: + - '@1.12.3' + pflogger: + require: + - +mpi + pixman: + require: + - +pic + prod-util: + require: + - '@2.1.1' + proj: + require: + - ~tiff + # Turn off crypt variant for Python; this leads to build errors + # with Intel in py-cryptography unless external curl and openssl + # are removed, which itself is problematic. + python: + require: + - '@3.11' + - ~crypt + py-cartopy: + require: + - +plotting + ## Pin py-cython to avoid duplicate packages + #py-cython: + # require: + # - '@3' + ## As long as we need the "delayed" variant for py-dask, + ## we are restricted to version 2021.03.0 (last one that has it) + ## and newer than 2021.01.0 because of these issues: + ## https://github.com/JCSDA/spack-stack/issues/1216 + ## https://github.com/pydata/xarray/issues/8917 + #py-dask: + # require: + # - '@2021.3.0' + ## To avoid duplicate packages + #py-flit-core: + # require: + # - '@3.8.0' + py-h5py: + require: + - ~mpi + # https://github.com/JCSDA/spack-stack/issues/1276 + #py-matplotlib: + # require: + # - '@3.7.4' + # https://github.com/Unidata/netcdf4-python/issues/1389 + # https://github.com/Unidata/netcdf4-python/issues/1389#issuecomment-2599760051 + py-netcdf4: + require: + - '@1.7.2' + - ~mpi + ## To avoid duplicate packages + #py-numpy: + # require: + # - '@1.26' + ## To avoid duplicate packages + #py-ruamel-yaml: + # require: + # - '@0.17.16' + ## Pin the py-setuptools version to avoid duplicate Python packages + #py-setuptools: + # require: + # - '@69' + ## When making changes here, also check the packages.yaml for atlantis, nautilus, blackpearl + py-torch: + require: + - +custom-protobuf ~mkldnn + ## To avoid duplicate packages + #py-urllib3: + # require: + # - '@1.26.12' + ## To avoid duplicate packages + #py-versioneer: + # require: + # - '@0.28' + qt: + require: + - '@5' + scotch: + require: + - '@7.0.4' + - +mpi+metis~shared~threads~mpi_thread+noarch+esmumps + sfcio: + require: + - '@1.4.2' + shumlib: + require: + - '@macos_clang_linux_intel_port' + sigio: + require: + - '@2.3.3' + sp: + require: + - '@2.5.0' + - 'precision=4,d,8' + udunits: + require: + - '@2.2.28' + ufs-utils: + require: + - '@1.13.0' + # Note - we can remove upp from stack at some point? + upp: + require: + - '@10.0.10' + # Need extradeps for grib-utils, enable by default to avoid duplicate packages + w3emc: + require: + - '@2.10.0' + - precision=4,d,8 +extradeps + w3nco: + require: + - '@2.4.1' + # See macOS site config for wgrib2 version/variant overrides + wgrib2: + require: + - '@3.6.0' + wrf-io: + require: + - '@1.2.0' + zstd: + require: + - '@1.5.7' + - +programs diff --git a/configs/common/packages_gcc.yaml b/configs/common/packages_gcc.yaml index 34b78df4f..4101c232d 100644 --- a/configs/common/packages_gcc.yaml +++ b/configs/common/packages_gcc.yaml @@ -1,26 +1,26 @@ packages: - c: - require: - - "gcc" - cxx: - require: - - "gcc" - fortran: - require: - - "gcc" - all: - require: - - '%[when=%c]gcc %[when=%cxx]gcc %[when=%fortran]gcc' - providers: - blas: [openblas] - fftw-api: [fftw] - lapack: [openblas] - ectrans: - require: - - '~mkl +fftw' - gsibec: - require: - - '~mkl' - py-numpy: - require: - - '^openblas' + c: + require: + - 'gcc' + cxx: + require: + - 'gcc' + fortran: + require: + - 'gcc' + all: + require: + - '%[when=%c]gcc %[when=%cxx]gcc %[when=%fortran]gcc' + providers: + blas: [openblas] + fftw-api: [fftw] + lapack: [openblas] + ectrans: + require: + - ~mkl +fftw + gsibec: + require: + - ~mkl + py-numpy: + require: + - '^openblas' diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index 01b46b3d3..21e163ca3 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -1,82 +1,94 @@ packages: - # Allow certain packages to be built with GCC as long as they only use - # C and/or C++. Because Fortran compilers are not ABI compatible, we - # need to force spack to use one and only one compiler (Intel oneAPI) - c: - require: - - one_of: ["intel-oneapi-compilers", "gcc"] - prefer: - - "intel-oneapi-compilers" - cxx: - require: - - one_of: ["intel-oneapi-compilers", "gcc"] - prefer: - - "intel-oneapi-compilers" - fortran: - require: - - "intel-oneapi-compilers" - all: - require: - - '%[when=%c]oneapi %[when=%cxx]oneapi %[when=%fortran]oneapi' - providers: - blas: [intel-oneapi-mkl] - fftw-api: [intel-oneapi-mkl] - lapack: [intel-oneapi-mkl] - antlr: - require: - - '%gcc' - bison: - require: - - '%gcc' - boost: - require: - - '%oneapi' - cmake: - require: - - '%gcc' - ectrans: - require: - - '+mkl ~fftw' - gmake: - require: - - '%gcc' - gsibec: - require: - - '+mkl' - libbsd: - require: - - '%gcc' - libmd: - require: - - '%gcc' - met: - require: - - '+shared-intel' - netcdf-c: - require: - - '%oneapi' - netcdf-cxx: - require: - - '%oneapi' - nco: - require: - - '%gcc' - p4est: - require: - - '%oneapi' - parallel-netcdf: - require: - - '+shared-intel' - python: - require: - - '%oneapi' - py-numpy: - require: - - '%oneapi' - - '^intel-oneapi-mkl' - qt: - require: - - '%gcc' - scotch: - require: - - '%gcc' + # Allow certain packages to be built with GCC as long as they only use + # C and/or C++. Because Fortran compilers are not ABI compatible, we + # need to force spack to use one and only one compiler (Intel oneAPI) + c: + require: + - one_of: ['intel-oneapi-compilers', 'gcc'] + prefer: + - 'intel-oneapi-compilers' + cxx: + require: + - one_of: ['intel-oneapi-compilers', 'gcc'] + prefer: + - 'intel-oneapi-compilers' + fortran: + require: + - 'intel-oneapi-compilers' + all: + require: + - '%[when=%c]oneapi %[when=%cxx]oneapi %[when=%fortran]oneapi' + providers: + blas: [intel-oneapi-mkl] + fftw-api: [intel-oneapi-mkl] + lapack: [intel-oneapi-mkl] + antlr: + require: + - '%gcc' + bison: + require: + - '%gcc' + boost: + require: + - '%oneapi' + cmake: + require: + - '%gcc' + ectrans: + require: + - +mkl ~fftw + gmake: + require: + - '%gcc' + gsibec: + require: + - +mkl + libbsd: + require: + - '%gcc' + libmd: + require: + - '%gcc' + met: + require: + - +shared-intel + netcdf-c: + require: + - '%oneapi' + netcdf-cxx: + require: + - '%oneapi' + nco: + require: + - '%gcc' + p4est: + require: + - '%oneapi' + parallel-netcdf: + require: + - +shared-intel + python: + require: + - '%gcc' + py-cython: + require: + - '%gcc' + py-numpy: + require: + - '%gcc' + - '^intel-oneapi-mkl' + py-pandas: + require: + - '%gcc' + py-scipy: + require: + - '%gcc' + py-xarray: + require: + - '%gcc' + qt: + require: + - '%gcc' + scotch: + require: + - '%gcc' From 9bd24b670ed82c1ffccef0277c489b8203cc11f6 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 5 Aug 2025 15:13:14 -0600 Subject: [PATCH 019/163] Update site config for Bounty --- .../{compilers.yaml => compilers.yaml.old} | 0 configs/sites/tier2/bounty/modules.yaml | 3 +- configs/sites/tier2/bounty/packages.yaml | 7 +++ .../tier2/bounty/packages_gcc-13.3.1.yaml | 17 ++++++++ configs/sites/tier2/bounty/packages_gcc.yaml | 9 ---- .../bounty/packages_oneapi-2025.1.1.yaml | 43 +++++++++++++++++++ .../sites/tier2/bounty/packages_oneapi.yaml | 23 ---------- 7 files changed, 68 insertions(+), 34 deletions(-) rename configs/sites/tier2/bounty/{compilers.yaml => compilers.yaml.old} (100%) create mode 100644 configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml delete mode 100644 configs/sites/tier2/bounty/packages_gcc.yaml create mode 100644 configs/sites/tier2/bounty/packages_oneapi-2025.1.1.yaml delete mode 100644 configs/sites/tier2/bounty/packages_oneapi.yaml diff --git a/configs/sites/tier2/bounty/compilers.yaml b/configs/sites/tier2/bounty/compilers.yaml.old similarity index 100% rename from configs/sites/tier2/bounty/compilers.yaml rename to configs/sites/tier2/bounty/compilers.yaml.old diff --git a/configs/sites/tier2/bounty/modules.yaml b/configs/sites/tier2/bounty/modules.yaml index 57e6be9fe..43f631874 100644 --- a/configs/sites/tier2/bounty/modules.yaml +++ b/configs/sites/tier2/bounty/modules.yaml @@ -5,6 +5,5 @@ modules: tcl: include: # List of packages for which we need modules that are blacklisted by default + - intel-oneapi-mpi - openmpi - - mpich - - python diff --git a/configs/sites/tier2/bounty/packages.yaml b/configs/sites/tier2/bounty/packages.yaml index a9c812bf7..2cdf9dbbe 100644 --- a/configs/sites/tier2/bounty/packages.yaml +++ b/configs/sites/tier2/bounty/packages.yaml @@ -1,4 +1,10 @@ packages: + # Modification of common packages + # Prevent building zlib, zlib-ng + zlib-api: + buildable: false + + # All other packages listed alphabetically autoconf: externals: - spec: autoconf@2.69 @@ -60,6 +66,7 @@ packages: - spec: groff@1.22.4 prefix: /usr m4: + # DH* NEEDED? buildable: false externals: - spec: m4@1.4.19 prefix: /usr diff --git a/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml b/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml new file mode 100644 index 000000000..05ca18a79 --- /dev/null +++ b/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml @@ -0,0 +1,17 @@ +packages: + mpi: + require: + - "openmpi@=5.0.6" + gcc: + externals: + - spec: gcc@13.3.1 languages:='c,c++,fortran' + prefix: /opt/rh/gcc-toolset-13/root/usr + extra_attributes: + compilers: + c: /opt/rh/gcc-toolset-13/root/usr/bin/gcc + cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ + fortran: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@13.3.1%gcc@13.3.1 + prefix: /opt/rh/gcc-toolset-13/root/usr diff --git a/configs/sites/tier2/bounty/packages_gcc.yaml b/configs/sites/tier2/bounty/packages_gcc.yaml deleted file mode 100644 index 6b6c843df..000000000 --- a/configs/sites/tier2/bounty/packages_gcc.yaml +++ /dev/null @@ -1,9 +0,0 @@ -packages: - all: - compiler:: [gcc@13.3.1] - providers: - mpi:: [openmpi@5.0.6] - gcc-runtime: - externals: - - spec: gcc-runtime@13.3.1%gcc@13.3.1 - prefix: /opt/rh/gcc-toolset-13/root/usr diff --git a/configs/sites/tier2/bounty/packages_oneapi-2025.1.1.yaml b/configs/sites/tier2/bounty/packages_oneapi-2025.1.1.yaml new file mode 100644 index 000000000..bf598e38e --- /dev/null +++ b/configs/sites/tier2/bounty/packages_oneapi-2025.1.1.yaml @@ -0,0 +1,43 @@ +packages: + mpi: + buildable: False + require: + - "intel-oneapi-mpi@2021.15" + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2025.1.1 + prefix: /opt/intel/oneapi + extra_attributes: + compilers: + c: /opt/intel/oneapi/2025.1/bin/icx + cxx: /opt/intel/oneapi/2025.1/bin/icpx + fortran: /opt/intel/oneapi/2025.1/bin/ifx + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.15 + prefix: /opt/intel/oneapi + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.1 + prefix: /opt/intel/oneapi + intel-oneapi-runtime: + # LINK TO BUG REPORT HERE + #buildable: false + externals: + - spec: intel-oneapi-runtime@2025.1.1 + prefix: /opt/intel/oneapi + gcc: + externals: + - spec: gcc@11.5.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@11.5.0%gcc@11.5.0 + prefix: /usr diff --git a/configs/sites/tier2/bounty/packages_oneapi.yaml b/configs/sites/tier2/bounty/packages_oneapi.yaml deleted file mode 100644 index fa10ea00e..000000000 --- a/configs/sites/tier2/bounty/packages_oneapi.yaml +++ /dev/null @@ -1,23 +0,0 @@ -packages: - all: - compiler:: [oneapi@2025.1.1, gcc@11.5.0] - providers: - mpi:: [intel-oneapi-mpi@2021.15] - mpi: - buildable: False - intel-oneapi-mpi: - externals: - - spec: intel-oneapi-mpi@2021.15%oneapi@2025.1.1 - prefix: /opt/intel/oneapi - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2025.1%oneapi@2025.1.1 - prefix: /opt/intel/oneapi - intel-oneapi-runtime: - externals: - - spec: intel-oneapi-runtime@2025.1.1%oneapi@2025.1.1 - prefix: /opt/intel/oneapi - gcc-runtime: - externals: - - spec: gcc-runtime@13.3.1%gcc@13.3.1 - prefix: /opt/rh/gcc-toolset-13/root/usr From 9ea3d0a2c0ce6a2dbd9459f175f7f1f97f6264c2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 5 Aug 2025 15:13:49 -0600 Subject: [PATCH 020/163] Update configs/common/{modules_lmod.yaml,modules_tcl.yaml,packages.yaml} --- configs/common/modules_lmod.yaml | 1 + configs/common/modules_tcl.yaml | 1 + configs/common/packages.yaml | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index fbbb75773..6b32dca46 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -36,6 +36,7 @@ modules: - googletest - hwloc - inputproto + - intel-oneapi-runtime - kbproto - libaec - libbsd diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index c6cae9461..c2980f0c8 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -34,6 +34,7 @@ modules: - googletest - hwloc - inputproto + - intel-oneapi-runtime - kbproto - libaec - libbsd diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index edc8100bc..72af2e0b1 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -84,7 +84,7 @@ packages: # Also, check the acorn and derecho site configs which have esmf modifications. esmf: require: - - ~xerces ~pnetcdf +shared +external-parallelio ~python + - ~xerces ~pnetcdf +shared +external-parallelio - any_of: ['@=8.6.1 snapshot=none', '@=8.8.0 snapshot=none', '@=8.9.0b12 snapshot=b12'] - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] when: '%intel' From eb0a254f363773cad23c1b5020c4f11bb1265c53 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 5 Aug 2025 15:14:39 -0600 Subject: [PATCH 021/163] spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py: use spack's COMPILER_TRANSLATION_TABLE --- .../lib/jcsda-emc/spack-stack/stack/meta_modules.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index e7077a625..eb807c739 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -178,10 +178,11 @@ def get_preferred_compiler(): if len(preferred_compilers)>1: raise Exception(f"Invalid value for packages:all:require is {preferred_compilers}") match = re.search(r'%\[when=%fortran\](\S+)', preferred_compilers[0]) - if match and match.group(1) in COMPILER_TRANSLATION_TABLE.keys(): - preferred_compiler_v1 = COMPILER_TRANSLATION_TABLE[match.group(1)] + # Translate legacy names (intel, oneapi, ...) to new names (intel-oneapi-compilers-classic, ...) + if match and match.group(1) in spack.aliases.LEGACY_COMPILER_TO_BUILTIN.keys(): + preferred_compiler_v1 = spack.aliases.LEGACY_COMPILER_TO_BUILTIN[match.group(1)] else: - raise Exception(f"Invalid value for packages:all:require {preferred_compilers[0]}") + preferred_compiler_v1 = match.group(1) # Method 2 try: preferred_compilers = spack.config.get("packages")["fortran"]["require"] @@ -192,10 +193,7 @@ def get_preferred_compiler(): ) if len(preferred_compilers)>1: raise Exception(f"Invalid value for packages:fortran:require is {preferred_compilers}") - if preferred_compilers[0] in COMPILER_TRANSLATION_TABLE.values(): - preferred_compiler_v2 = preferred_compilers[0] - else: - raise Exception(f"Invalid value for packages:fortran:require {preferred_compilers[0]}") + preferred_compiler_v2 = preferred_compilers[0] if preferred_compiler_v1 == preferred_compiler_v2: preferred_compiler = preferred_compiler_v1 del preferred_compilers From 4d7f82e4f7fc8e1ee2fa70a6fdf3343588242e3d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 5 Aug 2025 15:14:59 -0600 Subject: [PATCH 022/163] Temporarily comment out most Python dependencies in repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py --- .../packages/neptune_env/package.py | 4 +-- .../packages/neptune_python_env/package.py | 30 ++++++++----------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py index 1ea89b4fe..a6121fb56 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py @@ -34,8 +34,8 @@ class NeptuneEnv(BundlePackage): depends_on("p4est", type="run") depends_on("w3emc", type="run") depends_on("ip", type="run") - depends_on("esmf@8.9.0b12 ~debug ~python snapshot=b12", type="run", when="~debug") - depends_on("esmf@8.9.0b12 +debug ~python snapshot=b12", type="run", when="+debug") + depends_on("esmf@8.9.0b12 ~debug snapshot=b12", type="run", when="~debug") + depends_on("esmf@8.9.0b12 +debug snapshot=b12", type="run", when="+debug") depends_on("nco", type="run") depends_on("mct", type="run") diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py index 21e41d953..79c9eb260 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py @@ -22,27 +22,23 @@ class NeptunePythonEnv(BundlePackage): variant("xnrl", default=False, description="Build non-pulic XNRL") - #depends_on("c", type="run") - #depends_on("cxx", type="run") - #depends_on("fortran", type="run") - depends_on("neptune-env", type="run") # Enable the Python variant for ESMF depends_on("esmf +python", type="run") - depends_on("py-arch", type="run") - depends_on("py-h5py", type="run") - depends_on("py-netcdf4", type="run") - depends_on("py-pandas", type="run") - depends_on("py-pycodestyle", type="run") - depends_on("py-pybind11", type="run") - depends_on("py-pyhdf", type="run") - depends_on("py-pyyaml", type="run") - depends_on("py-regionmask", type="run") - depends_on("py-scipy", type="run") - #depends_on("py-xarray", type="run") - depends_on("py-pytest", type="run") - depends_on("py-fortranformat", type="run") + #depends_on("py-arch", type="run") + #depends_on("py-h5py", type="run") + #depends_on("py-netcdf4", type="run") + #depends_on("py-pandas", type="run") + #depends_on("py-pycodestyle", type="run") + #depends_on("py-pybind11", type="run") + #depends_on("py-pyhdf", type="run") + #depends_on("py-pyyaml", type="run") + #depends_on("py-regionmask", type="run") + #depends_on("py-scipy", type="run") + ##depends_on("py-xarray", type="run") + #depends_on("py-pytest", type="run") + #depends_on("py-fortranformat", type="run") #depends_on("met", type="run") #depends_on("metplus", type="run") From e2063a5958f681f2393745cf38121a830bf30782 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 7 Aug 2025 05:58:22 -0600 Subject: [PATCH 023/163] Add py-cython to module exclude list --- configs/common/modules_lmod.yaml | 1 + configs/common/modules_tcl.yaml | 1 + repos/builtin | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 6b32dca46..93104921f 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -76,6 +76,7 @@ modules: - perl - pkgconf - pkg-config + - py-cython - qt - randrproto - readline diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index c2980f0c8..2ff51d8fe 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -74,6 +74,7 @@ modules: - perl - pkgconf - pkg-config + - py-cython - qt - randrproto - readline diff --git a/repos/builtin b/repos/builtin index eacd94be2..e5ff008c8 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit eacd94be26e036e5f28d4db6cde8d496772f207e +Subproject commit e5ff008c85bc4374c9a6fd70887eaebe8c07d584 From 9fb97ec295e4d5e54112ffda4dabcc39669f4948 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 7 Aug 2025 06:00:04 -0600 Subject: [PATCH 024/163] configs/common/packages.yaml: update boost and netcdf-c --- configs/common/packages.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 72af2e0b1..74256a066 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -14,13 +14,11 @@ packages: bison: require: - '@3.4:' - # 1.85 incompatible with ecflow@5.11.4 - use latest 'good version' boost: require: - - '@1.84' - ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log - ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization - - ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden + - +system +test +thread +timer ~wave cxxstd=17 visibility=hidden bufr: require: - '@12.1.0' @@ -238,7 +236,7 @@ packages: netcdf-c: require: - '@4.9.2' - - +dap +mpi ~parallel-netcdf ~szip +parallel_tests build_system=autotools + - +dap +mpi ~parallel-netcdf ~szip +parallel_tests netcdf-cxx4: require: - '@4.3.1' From 0649cc38c706a4faa6200b120f6e6b847ddd3d4d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 7 Aug 2025 06:01:27 -0600 Subject: [PATCH 025/163] Remove unnecessary entries in configs/common/packages_oneapi.yaml --- configs/common/packages_oneapi.yaml | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index 21e163ca3..c98ba29c1 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -28,6 +28,7 @@ packages: bison: require: - '%gcc' + # Not sure this is still needed? boost: require: - '%oneapi' @@ -52,40 +53,27 @@ packages: met: require: - +shared-intel + # Not sure this is still needed? netcdf-c: require: - '%oneapi' + # Not sure this is still needed? netcdf-cxx: require: - '%oneapi' nco: require: - '%gcc' + # Not sure this is still needed? p4est: require: - '%oneapi' parallel-netcdf: require: - +shared-intel - python: - require: - - '%gcc' - py-cython: - require: - - '%gcc' py-numpy: require: - - '%gcc' - '^intel-oneapi-mkl' - py-pandas: - require: - - '%gcc' - py-scipy: - require: - - '%gcc' - py-xarray: - require: - - '%gcc' qt: require: - '%gcc' From 5b0341ffa5909792f0f8b7befa147c43829d0855 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Aug 2025 16:33:39 -0600 Subject: [PATCH 026/163] Update configs/** and repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py --- configs/common/packages.yaml | 8 +++- configs/common/packages_gcc.yaml | 16 +++---- configs/common/packages_oneapi.yaml | 43 +++---------------- configs/templates/cylc-dev/spack.yaml | 14 ++---- repos/builtin | 2 +- .../packages/neptune_python_env/package.py | 38 ++++++++-------- 6 files changed, 41 insertions(+), 80 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 74256a066..2693bac32 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -236,7 +236,7 @@ packages: netcdf-c: require: - '@4.9.2' - - +dap +mpi ~parallel-netcdf ~szip +parallel_tests + - +dap +mpi ~parallel-netcdf ~szip +parallel_tests build_system=cmake netcdf-cxx4: require: - '@4.3.1' @@ -255,6 +255,7 @@ packages: openssl: require: - +shared + - '@3' p4est: require: - '@2.8.7' @@ -320,7 +321,10 @@ packages: #py-numpy: # require: # - '@1.26' - ## To avoid duplicate packages + # To avoid duplicate packages + py-python-dateutil: + require: + - '@2' #py-ruamel-yaml: # require: # - '@0.17.16' diff --git a/configs/common/packages_gcc.yaml b/configs/common/packages_gcc.yaml index 4101c232d..ee1a9e600 100644 --- a/configs/common/packages_gcc.yaml +++ b/configs/common/packages_gcc.yaml @@ -1,16 +1,7 @@ packages: - c: - require: - - 'gcc' - cxx: - require: - - 'gcc' - fortran: - require: - - 'gcc' all: - require: - - '%[when=%c]gcc %[when=%cxx]gcc %[when=%fortran]gcc' + prefer: + - '%gcc' providers: blas: [openblas] fftw-api: [fftw] @@ -21,6 +12,9 @@ packages: gsibec: require: - ~mkl + ip: + require: + - '^openblas' py-numpy: require: - '^openblas' diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index c98ba29c1..a42eeb486 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -1,23 +1,7 @@ packages: - # Allow certain packages to be built with GCC as long as they only use - # C and/or C++. Because Fortran compilers are not ABI compatible, we - # need to force spack to use one and only one compiler (Intel oneAPI) - c: - require: - - one_of: ['intel-oneapi-compilers', 'gcc'] - prefer: - - 'intel-oneapi-compilers' - cxx: - require: - - one_of: ['intel-oneapi-compilers', 'gcc'] - prefer: - - 'intel-oneapi-compilers' - fortran: - require: - - 'intel-oneapi-compilers' all: - require: - - '%[when=%c]oneapi %[when=%cxx]oneapi %[when=%fortran]oneapi' + prefer: + - '%oneapi' providers: blas: [intel-oneapi-mkl] fftw-api: [intel-oneapi-mkl] @@ -28,10 +12,6 @@ packages: bison: require: - '%gcc' - # Not sure this is still needed? - boost: - require: - - '%oneapi' cmake: require: - '%gcc' @@ -44,6 +24,9 @@ packages: gsibec: require: - +mkl + ip: + require: + - '^intel-oneapi-mkl' libbsd: require: - '%gcc' @@ -53,21 +36,9 @@ packages: met: require: - +shared-intel - # Not sure this is still needed? - netcdf-c: - require: - - '%oneapi' - # Not sure this is still needed? - netcdf-cxx: - require: - - '%oneapi' nco: require: - '%gcc' - # Not sure this is still needed? - p4est: - require: - - '%oneapi' parallel-netcdf: require: - +shared-intel @@ -77,6 +48,4 @@ packages: qt: require: - '%gcc' - scotch: - require: - - '%gcc' + diff --git a/configs/templates/cylc-dev/spack.yaml b/configs/templates/cylc-dev/spack.yaml index b22c13ea4..1370b9dfb 100644 --- a/configs/templates/cylc-dev/spack.yaml +++ b/configs/templates/cylc-dev/spack.yaml @@ -20,14 +20,8 @@ spack: link_type: symlink include: [] - definitions: - - compilers: ['%gcc'] - - packages: - - py-cylc-flow@8.4.2 - - py-cylc-rose@1.5.1 - - py-cylc-uiserver@1.6.1 - specs: - - matrix: - - [$packages] - - [$compilers] + - py-cylc-flow@8.4.2 + - py-cylc-rose@1.5.1 + - py-cylc-uiserver@1.6.1 + diff --git a/repos/builtin b/repos/builtin index e5ff008c8..406275a50 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit e5ff008c85bc4374c9a6fd70887eaebe8c07d584 +Subproject commit 406275a503e9ffa053492873133e69fc0dffa4f3 diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py index 79c9eb260..605ac0b59 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py @@ -26,24 +26,24 @@ class NeptunePythonEnv(BundlePackage): # Enable the Python variant for ESMF depends_on("esmf +python", type="run") - #depends_on("py-arch", type="run") - #depends_on("py-h5py", type="run") - #depends_on("py-netcdf4", type="run") - #depends_on("py-pandas", type="run") - #depends_on("py-pycodestyle", type="run") - #depends_on("py-pybind11", type="run") - #depends_on("py-pyhdf", type="run") - #depends_on("py-pyyaml", type="run") - #depends_on("py-regionmask", type="run") - #depends_on("py-scipy", type="run") - ##depends_on("py-xarray", type="run") - #depends_on("py-pytest", type="run") - #depends_on("py-fortranformat", type="run") - - #depends_on("met", type="run") - #depends_on("metplus", type="run") - - #with when("+xnrl"): - # depends_on("py-xnrl", type="run") + depends_on("py-arch", type="run") + depends_on("py-h5py", type="run") + depends_on("py-netcdf4", type="run") + depends_on("py-pandas", type="run") + depends_on("py-pycodestyle", type="run") + depends_on("py-pybind11", type="run") + depends_on("py-pyhdf", type="run") + depends_on("py-pyyaml", type="run") + depends_on("py-regionmask", type="run") + depends_on("py-scipy", type="run") + depends_on("py-xarray", type="run") + depends_on("py-pytest", type="run") + depends_on("py-fortranformat", type="run") + + depends_on("met", type="run") + depends_on("metplus", type="run") + + with when("+xnrl"): + depends_on("py-xnrl", type="run") # There is no need for install() since there is no code. From 4d4e671e5d6273cf9fcbb8176ef3aa4db005e1c2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Aug 2025 16:33:49 -0600 Subject: [PATCH 027/163] Remove filter_compilers from spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py --- .../jcsda-emc/spack-stack/stack/stack_env.py | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index df5857e0f..eb59821d1 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -229,21 +229,6 @@ def get_upstream_realpaths(self, upstream_path): upstream_paths += self.get_upstream_realpaths(entry[1]["install_tree"]) return upstream_paths - def filter_site_compilers(self, compilers_yaml_path, compiler_to_keep): - compiler_to_keep = re.sub("@=", "@", compiler_to_keep) - config_compiler_name = re.sub("@.*", "", compiler_to_keep) - with open(compilers_yaml_path, "r") as f: - compilers_yaml = syaml.load_config(f) - n_compilers = len(compilers_yaml["compilers"]) - for i in range(n_compilers-1, -1, -1): - spec_tmp = compilers_yaml["compilers"][i]["compiler"]["spec"] - spec_tmp = re.sub("@=", "@", spec_tmp) - name_tmp = re.sub("@.*", "", spec_tmp) - if (name_tmp == config_compiler_name) and (spec_tmp != compiler_to_keep): - del(compilers_yaml["compilers"][i]) - with open(compilers_yaml_path, "w") as f: - syaml.dump_config(compilers_yaml, stream=f) - def write(self): """Write environment out to a spack.yaml in /. Will create env_dir if it does not exist. @@ -259,12 +244,6 @@ def write(self): # Copy site config first so that it takes precedence in env if self.site != "none": self._copy_site_includes() - # DH* - print("WARNING, filter_site_compilers currently commented out") - ## By default remove all other versions of the specified compiler - #if not self.keepallcompilers and "@" in self.compiler: - # self.filter_site_compilers(os.path.join(env_dir, "site", "compilers.yaml"), self.compiler) - # *DH # Copy common include files self._copy_common_includes() From 9678b8ca9e550f235a5834408df541d6ddbcd9fb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 12 Aug 2025 17:04:19 -0600 Subject: [PATCH 028/163] configs/common/packages.yaml: Update so that neptune-python-env builds with oneapi@2025.1.1 --- configs/common/packages.yaml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 2693bac32..425e71f6b 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -82,7 +82,7 @@ packages: # Also, check the acorn and derecho site configs which have esmf modifications. esmf: require: - - ~xerces ~pnetcdf +shared +external-parallelio + - ~xerces ~pnetcdf +shared +external-parallelio +python - any_of: ['@=8.6.1 snapshot=none', '@=8.8.0 snapshot=none', '@=8.9.0b12 snapshot=b12'] - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] when: '%intel' @@ -125,6 +125,11 @@ packages: gcc-runtime: require: - '%gcc' + # This version builds with Intel oneAPI compilers, newer/older versions don't + gdal: + require: + - '@3.10.2' + # *DH gfsio: require: - '@1.4.2' @@ -288,6 +293,10 @@ packages: py-cartopy: require: - +plotting + # Pin to version 42 to avoid maturin dependency that breaks Intel oneAPI builds + py-cryptography: + require: + - '@42' ## Pin py-cython to avoid duplicate packages #py-cython: # require: @@ -321,6 +330,10 @@ packages: #py-numpy: # require: # - '@1.26' + # This version builds with Intel oneAPI compilers, newer versions don't + py-pyogrio: + require: + - '@0.9.0' # To avoid duplicate packages py-python-dateutil: require: @@ -328,10 +341,10 @@ packages: #py-ruamel-yaml: # require: # - '@0.17.16' - ## Pin the py-setuptools version to avoid duplicate Python packages - #py-setuptools: - # require: - # - '@69' + # Pin the py-setuptools version to avoid duplicate Python packages + py-setuptools: + require: + - '@69' ## When making changes here, also check the packages.yaml for atlantis, nautilus, blackpearl py-torch: require: From cfecd5761a6517eb89fc3d6f17f54748cf58ab0b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 Aug 2025 08:46:46 -0600 Subject: [PATCH 029/163] Add jedi-neptune-env and crtm to configs/templates/neptune-dev --- configs/templates/neptune-dev/spack.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 32a73ca36..94caed3ed 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -9,8 +9,12 @@ spack: - neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 - neptune-env +debug +espc ^esmf@=8.9.0b12 snapshot=b12 - neptune-python-env +xnrl ^neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 + - jedi-neptune-env + - crtm@3.1.2 + - crtm@v2.4.1-jedi.2 + packages: # Avoid duplicate builds of parallelio +pnetcdf and ~pnetcdf parallelio: require: - - '~pnetcdf' + - ~pnetcdf From 7d1c0ad7ce199540469e35354b1a798e68f3a9b3 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 Aug 2025 08:47:08 -0600 Subject: [PATCH 030/163] Add py-poetry-core to list of excluded modules in configs/common/modules_* --- configs/common/modules_lmod.yaml | 1 + configs/common/modules_tcl.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 93104921f..410275773 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -77,6 +77,7 @@ modules: - pkgconf - pkg-config - py-cython + - py-poetry-core - qt - randrproto - readline diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 2ff51d8fe..3df12ea1f 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -75,6 +75,7 @@ modules: - pkgconf - pkg-config - py-cython + - py-poetry-core - qt - randrproto - readline From f7c290b4a07bde9809f7720f3f528c3cf68aab34 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 Aug 2025 08:48:10 -0600 Subject: [PATCH 031/163] Update configs/common/packages*.yaml for building jedi-neptune-env as part of neptune-dev --- configs/common/packages.yaml | 6 ++++++ configs/common/packages_gcc.yaml | 3 --- configs/common/packages_oneapi.yaml | 3 --- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 425e71f6b..40d2f42e6 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -330,6 +330,12 @@ packages: #py-numpy: # require: # - '@1.26' + # DH* 20250813 + # To avoid duplicate packages + py-poetry-core: + require: + - '@1.7' + # *DH 20250813 # This version builds with Intel oneAPI compilers, newer versions don't py-pyogrio: require: diff --git a/configs/common/packages_gcc.yaml b/configs/common/packages_gcc.yaml index ee1a9e600..82726d914 100644 --- a/configs/common/packages_gcc.yaml +++ b/configs/common/packages_gcc.yaml @@ -9,9 +9,6 @@ packages: ectrans: require: - ~mkl +fftw - gsibec: - require: - - ~mkl ip: require: - '^openblas' diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index a42eeb486..4f88b7f69 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -21,9 +21,6 @@ packages: gmake: require: - '%gcc' - gsibec: - require: - - +mkl ip: require: - '^intel-oneapi-mkl' From 234b2c5fdbc5dfe5f2c50c803037f0e03fa038c9 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 Aug 2025 08:48:32 -0600 Subject: [PATCH 032/163] Comment out 'sp' in repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env/package.py --- .../spack_repo/spack_stack/packages/jedi_base_env/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env/package.py index 08317d920..eb5914940 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env/package.py @@ -49,7 +49,7 @@ class JediBaseEnv(BundlePackage): depends_on("nlohmann-json", type="run") depends_on("nlohmann-json-schema-validator", type="run") depends_on("odc", type="run") - depends_on("sp", type="run", when="^ip@:4") + #depends_on("sp", type="run", when="^ip@:4") depends_on("udunits", type="run") # Python packages From e43b78d0ed6c0580f60992797faab5abd7209aa8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 Aug 2025 08:48:48 -0600 Subject: [PATCH 033/163] Update finding preferred compiler in spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py --- .../spack-stack/stack/meta_modules.py | 68 +++++++++++-------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index eb807c739..90351a102 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -167,40 +167,52 @@ def get_preferred_compiler(): """DH* 20250730 - DO WE NEED BOTH WAYS TO DETERMINE THE PREFERRED COMPILER? # MAYBE KEEP FOR NOW UNTIL WE KNOW BETTER WHAT IS NEEDED AND WHAT NOT. TODO: UPDATE DOCSTRING""" - # Method 1 - try: - preferred_compilers = spack.config.get("packages")["all"]["require"] - except: - raise Exception( - """Unable to detect preferred compiler from environment. - Does the environment have the config entry 'packages:all:require?'""" - ) - if len(preferred_compilers)>1: - raise Exception(f"Invalid value for packages:all:require is {preferred_compilers}") - match = re.search(r'%\[when=%fortran\](\S+)', preferred_compilers[0]) - # Translate legacy names (intel, oneapi, ...) to new names (intel-oneapi-compilers-classic, ...) - if match and match.group(1) in spack.aliases.LEGACY_COMPILER_TO_BUILTIN.keys(): - preferred_compiler_v1 = spack.aliases.LEGACY_COMPILER_TO_BUILTIN[match.group(1)] - else: - preferred_compiler_v1 = match.group(1) - # Method 2 + ## Method 1 + #try: + # preferred_compilers = spack.config.get("packages")["all"]["require"] + #except: + # raise Exception( + # """Unable to detect preferred compiler from environment. + # Does the environment have the config entry 'packages:all:require?'""" + # ) + #if len(preferred_compilers)>1: + # raise Exception(f"Invalid value for packages:all:require is {preferred_compilers}") + #match = re.search(r'%\[when=%fortran\](\S+)', preferred_compilers[0]) + ## Translate legacy names (intel, oneapi, ...) to new names (intel-oneapi-compilers-classic, ...) + #if match and match.group(1) in spack.aliases.LEGACY_COMPILER_TO_BUILTIN.keys(): + # preferred_compiler_v1 = spack.aliases.LEGACY_COMPILER_TO_BUILTIN[match.group(1)] + #else: + # preferred_compiler_v1 = match.group(1) + ## Method 2 + #try: + # preferred_compilers = spack.config.get("packages")["fortran"]["require"] + #except: + # raise Exception( + # """Unable to detect preferred compiler from environment. + # Does the environment have the config entry 'packages:fortran:require?'""" + # ) + #if len(preferred_compilers)>1: + # raise Exception(f"Invalid value for packages:fortran:require is {preferred_compilers}") + #preferred_compiler_v2 = preferred_compilers[0] + #if preferred_compiler_v1 == preferred_compiler_v2: + # preferred_compiler = preferred_compiler_v1 + # del preferred_compilers + # del preferred_compiler_v1 + # del preferred_compiler_v2 + #else: + # raise Exception(f"Multiple preferred compilers in spack config: {preferred_compiler_v1} vs {preferred_compiler_v2}") + #return preferred_compiler + # Determine preferred compiler try: - preferred_compilers = spack.config.get("packages")["fortran"]["require"] + preferred_compilers = spack.config.get("packages")["all"]["prefer"] except: raise Exception( """Unable to detect preferred compiler from environment. - Does the environment have the config entry 'packages:fortran:require?'""" + Does the environment have the config entry 'packages:all:prefer?'""" ) if len(preferred_compilers)>1: - raise Exception(f"Invalid value for packages:fortran:require is {preferred_compilers}") - preferred_compiler_v2 = preferred_compilers[0] - if preferred_compiler_v1 == preferred_compiler_v2: - preferred_compiler = preferred_compiler_v1 - del preferred_compilers - del preferred_compiler_v1 - del preferred_compiler_v2 - else: - raise Exception(f"Multiple preferred compilers in spack config: {preferred_compiler_v1} vs {preferred_compiler_v2}") + raise Exception(f"Invalid value for packages:all:prefer is {preferred_compilers}") + preferred_compiler = preferred_compilers[0] return preferred_compiler From c3434be015255d7b70581d99994494e7c1564ec4 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 Aug 2025 08:28:40 -0600 Subject: [PATCH 034/163] Comment out gsibec and sp in repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env/package.py --- repos/builtin | 2 +- .../spack_repo/spack_stack/packages/jedi_base_env/package.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/repos/builtin b/repos/builtin index 406275a50..937a00428 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 406275a503e9ffa053492873133e69fc0dffa4f3 +Subproject commit 937a0042847d8c2ee3a7cc126d78f33d868049ff diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env/package.py index eb5914940..3043df8f8 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_base_env/package.py @@ -40,7 +40,8 @@ class JediBaseEnv(BundlePackage): depends_on("fftw-api", when="+fftw", type="run") depends_on("flex", type="run") depends_on("git-lfs", type="run") - depends_on("gsibec", type="run") + # Commented out until update to ip is completed + #depends_on("gsibec", type="run") depends_on("gsl-lite", type="run") depends_on("hdf", when="+hdf4", type="run") depends_on("jedi-cmake", type="run") From 39c4b1c9ecd011806c491478b99f6c330e8a07aa Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 Aug 2025 08:29:35 -0600 Subject: [PATCH 035/163] Turn off CI testing on macOS and Ubuntu --- .github/workflows/{ => TMP_OFF}/macos-ci-aarch64.yaml | 0 .github/workflows/{ => TMP_OFF}/ubuntu-ci-containers-x86_64.yaml | 0 .github/workflows/{ => TMP_OFF}/ubuntu-ci-x86_64-gnu.yaml | 0 .github/workflows/{ => TMP_OFF}/ubuntu-ci-x86_64-oneapi-ifx.yaml | 0 .github/workflows/{ => TMP_OFF}/ubuntu-ci-x86_64-oneapi.yaml | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ => TMP_OFF}/macos-ci-aarch64.yaml (100%) rename .github/workflows/{ => TMP_OFF}/ubuntu-ci-containers-x86_64.yaml (100%) rename .github/workflows/{ => TMP_OFF}/ubuntu-ci-x86_64-gnu.yaml (100%) rename .github/workflows/{ => TMP_OFF}/ubuntu-ci-x86_64-oneapi-ifx.yaml (100%) rename .github/workflows/{ => TMP_OFF}/ubuntu-ci-x86_64-oneapi.yaml (100%) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/TMP_OFF/macos-ci-aarch64.yaml similarity index 100% rename from .github/workflows/macos-ci-aarch64.yaml rename to .github/workflows/TMP_OFF/macos-ci-aarch64.yaml diff --git a/.github/workflows/ubuntu-ci-containers-x86_64.yaml b/.github/workflows/TMP_OFF/ubuntu-ci-containers-x86_64.yaml similarity index 100% rename from .github/workflows/ubuntu-ci-containers-x86_64.yaml rename to .github/workflows/TMP_OFF/ubuntu-ci-containers-x86_64.yaml diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-gnu.yaml similarity index 100% rename from .github/workflows/ubuntu-ci-x86_64-gnu.yaml rename to .github/workflows/TMP_OFF/ubuntu-ci-x86_64-gnu.yaml diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml b/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi-ifx.yaml similarity index 100% rename from .github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml rename to .github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi-ifx.yaml diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml similarity index 100% rename from .github/workflows/ubuntu-ci-x86_64-oneapi.yaml rename to .github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml From 19465afd27544af0eb8ab3c3d5d22853a12b2516 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 Aug 2025 14:24:24 -0600 Subject: [PATCH 036/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 937a00428..8faec0553 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 937a0042847d8c2ee3a7cc126d78f33d868049ff +Subproject commit 8faec05536c7bae79c33c1ae5ba5404631dfbef5 From ecbc63b1f2fccd9766b5032e81b8caf47e9727ff Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 Aug 2025 08:10:07 -0700 Subject: [PATCH 037/163] [skip ci] Update configs/** --- configs/common/packages.yaml | 16 ++-- .../tier1/atlantis/packages_gcc-13.4.0.yaml | 11 +-- configs/templates/neptune-dev/spack.yaml | 2 +- configs/templates/unified-dev/spack.yaml | 84 +++++++------------ 4 files changed, 45 insertions(+), 68 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 5cd81f6ce..440abd19c 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -16,6 +16,7 @@ packages: - '@3.4:' boost: require: + - '@1.84.0' - ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log - ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization - +system +test +thread +timer ~wave cxxstd=17 visibility=hidden @@ -310,17 +311,18 @@ packages: #py-dask: # require: # - '@2021.3.0' - ## To avoid duplicate packages - #py-flit-core: - # require: - # - '@3.8.0' + # To avoid duplicate packages + py-flit-core: + require: + - '@3.8.0' py-h5py: require: - ~mpi # https://github.com/JCSDA/spack-stack/issues/1276 - #py-matplotlib: - # require: - # - '@3.7.4' + py-matplotlib: + require: + - '@3.7.4' + #- '@3.8.0' # https://github.com/Unidata/netcdf4-python/issues/1389 # https://github.com/Unidata/netcdf4-python/issues/1389#issuecomment-2599760051 py-netcdf4: diff --git a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml index 632b6ba2e..5566679c9 100644 --- a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml +++ b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml @@ -4,11 +4,10 @@ packages: openmpi: buildable: False externals: - - spec: openmpi@4.1.5 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm %gcc@=13.4.0 - # Note: slurm must be loaded before openmpi on Atlantis + - spec: openmpi@4.1.8 %gcc@=13.4.0 + prefix: /gpfs/neptune/spack-stack/openmpi-4.1.8/gcc-13.4.0 modules: - - slurm - - openmpi/mlnx/gcc/64/4.1.5a1 + - openmpi/4.1.8 gcc: externals: - spec: gcc@13.4.0 languages:='c,c++,fortran' @@ -20,10 +19,6 @@ packages: c: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/gcc cxx: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/g++ fortran: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/gfortran - extra_rpaths: - # For compiling parallel-netcdf; Bright must have removed - # some symbolic links required for OpenMPI 4.1.5a1 ... - - /gpfs/neptune/spack-stack/openmpi-4.1.5-fix-lib gcc-runtime: externals: - spec: gcc-runtime@13.4.0%gcc@13.4.0 diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 94caed3ed..6e71e3e66 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -9,7 +9,7 @@ spack: - neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 - neptune-env +debug +espc ^esmf@=8.9.0b12 snapshot=b12 - neptune-python-env +xnrl ^neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 - - jedi-neptune-env + - jedi-neptune-env ^neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 - crtm@3.1.2 - crtm@v2.4.1-jedi.2 diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index a0bc717a2..335f7c835 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -5,63 +5,43 @@ spack: view: false include: [] - definitions: - - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - - packages: - - ewok-env +ecflow ~cylc - - ai-env - - geos-gcm-env ^esmf@=8.6.1 - - global-workflow-env ^esmf@=8.6.1 ^crtm@=3.1.2 - - gmao-swell-env ^crtm@=v2.4.1-jedi.2 - - gsi-env ^crtm@=3.1.2 - - jedi-fv3-env - - jedi-geos-env ^esmf@=8.6.1 - - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.9.0b12 - - jedi-tools-env - - jedi-ufs-env ^esmf@=8.8.0 - - jedi-um-env - - neptune-env ^esmf@=8.9.0b12 - - neptune-python-env ^esmf@=8.9.0b12 - - soca-env - - ufs-srw-app-env ^esmf@=8.8.0 ^crtm@=3.1.2 - - ufs-weather-model-env ^esmf@=8.8.0 ^crtm@=3.1.2 + specs: + - ewok-env +ecflow ~cylc + #- ai-env + - geos-gcm-env ^esmf@=8.6.1 + - global-workflow-env ^esmf@=8.6.1 ^crtm@=3.1.2 + - gmao-swell-env ^crtm@=v2.4.1-jedi.2 + - gsi-env ^crtm@=3.1.2 + - jedi-fv3-env + - jedi-geos-env ^esmf@=8.6.1 + - jedi-mpas-env + - jedi-neptune-env ^esmf@=8.9.0b12 + #- jedi-tools-env + - jedi-ufs-env ^esmf@=8.8.0 + - jedi-um-env + - neptune-env ^esmf@=8.9.0b12 + - neptune-python-env ^esmf@=8.9.0b12 + - soca-env + - ufs-srw-app-env ^esmf@=8.8.0 ^crtm@=3.1.2 + - ufs-weather-model-env ^esmf@=8.8.0 ^crtm@=3.1.2 - # Various crtm tags (list all to avoid duplicate packages) - - crtm@2.4.0.1 - - crtm@v2.4.1-jedi.2 - - crtm@3.1.2 + # Various crtm tags (list all to avoid duplicate packages) + #- crtm@2.4.0.1 + - crtm@v2.4.1-jedi.2 + - crtm@3.1.2 - # Mapl with various esmf tags - - mapl@2.53.0 ^esmf@8.6.1 - - mapl@2.53.4 ^esmf@8.8.0 + # Mapl with various esmf tags + - mapl@2.53.0 ^esmf@8.6.1 + - mapl@2.53.4 ^esmf@8.8.0 - # Various esmf tags (list all to avoid duplicate packages) - - esmf@=8.6.1 snapshot=none - - esmf@=8.8.0 snapshot=none - - esmf@=8.9.0b12 snapshot=b12 + # Various esmf tags (list all to avoid duplicate packages) + - esmf@=8.6.1 snapshot=none + - esmf@=8.8.0 snapshot=none + - esmf@=8.9.0b12 snapshot=b12 - # MADIS for WCOSS2 decoders. - - madis@4.5 + # MADIS for WCOSS2 decoders. + - madis@4.5 - specs: - - matrix: - - [$packages] - - [$compilers] - exclude: - # Don't build ai-env and jedi-tools-env with Intel or oneAPI, - # some packages don't build (e.g., py-torch in ai-env doesn't - # build with Intel, and there are constant problems concretizing - # the environment - - ai-env%intel - - ai-env%oneapi - - jedi-tools-env%intel - - jedi-tools-env%oneapi - # Skip neptune-python-env with Intel Classic due to problems - # with new versions of py-numpy, py-scipy, ... - - neptune-python-env%intel - # Skip incompatible CRTM versions with oneapi - - crtm@2.4.0.1%oneapi packages: # Avoid duplicate builds of parallelio +pnetcdf and ~pnetcdf parallelio: From 23c920321e06a86cf2e2a5ec98119e3b5ed9d125 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 Aug 2025 09:16:24 -0600 Subject: [PATCH 038/163] Fix invalid yaml syntax in configs/sites/tier1/ursa/packages_hpcx.yaml --- configs/sites/tier1/ursa/packages_hpcx.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/configs/sites/tier1/ursa/packages_hpcx.yaml b/configs/sites/tier1/ursa/packages_hpcx.yaml index 03b394646..b581b7891 100644 --- a/configs/sites/tier1/ursa/packages_hpcx.yaml +++ b/configs/sites/tier1/ursa/packages_hpcx.yaml @@ -15,7 +15,6 @@ packages: modules: - hpc-x/2.18.1-oneapi2024 prefix: /apps/hpcx-v2.18.1-gcc-mlnx_ofed-redhat9-cuda12-x86_64/ompi-oneapi2024 - buildable: False intel-oneapi-runtime: externals: - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 From 5e1c465c68bcd646ad1273c1bebb1ceaa58effe1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 Aug 2025 13:23:28 -0600 Subject: [PATCH 039/163] Add ~openmp variant for package ip with clang/apple-clang --- configs/common/packages_apple-clang.yaml | 3 + configs/common/packages_clang.yaml | 3 + .../spack-stack/stack/meta_modules.py | 8 +- .../jcsda-emc/spack-stack/stack/stack_env.py | 38 +++---- .../tests/test_setup_meta_modules.py | 99 +++++++++++-------- .../spack-stack/tests/test_stack_create.py | 73 +++++++++----- 6 files changed, 139 insertions(+), 85 deletions(-) diff --git a/configs/common/packages_apple-clang.yaml b/configs/common/packages_apple-clang.yaml index 70012d1a0..9fa764bc2 100644 --- a/configs/common/packages_apple-clang.yaml +++ b/configs/common/packages_apple-clang.yaml @@ -10,6 +10,9 @@ packages: gsibec: require: - '~mkl' + ip: + require: + - '~openmp' py-numpy: require: - '^openblas' diff --git a/configs/common/packages_clang.yaml b/configs/common/packages_clang.yaml index 70012d1a0..9fa764bc2 100644 --- a/configs/common/packages_clang.yaml +++ b/configs/common/packages_clang.yaml @@ -10,6 +10,9 @@ packages: gsibec: require: - '~mkl' + ip: + require: + - '~openmp' py-numpy: require: - '^openblas' diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 90351a102..4bdfecc2c 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -212,7 +212,11 @@ def get_preferred_compiler(): ) if len(preferred_compilers)>1: raise Exception(f"Invalid value for packages:all:prefer is {preferred_compilers}") - preferred_compiler = preferred_compilers[0] + preferred_compiler_legacy_name = preferred_compilers[0].replace('%','') + if preferred_compiler_legacy_name in spack.aliases.LEGACY_COMPILER_TO_BUILTIN.keys(): + preferred_compiler = spack.aliases.LEGACY_COMPILER_TO_BUILTIN[preferred_compiler_legacy_name] + else: + preferred_compiler = preferred_compiler_legacy_name return preferred_compiler @@ -300,6 +304,8 @@ def setup_meta_modules(): specs = spack.store.STORE.db.query(hashes=hashes) q = ProviderIndex(specs=specs, repository=spack.repo.PATH) + print(f"DH DEBUG: specs = {specs}") + c_providers = q.providers_for("c") cxx_providers = q.providers_for("cxx") fortran_providers = q.providers_for("fortran") diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index eb59821d1..8787d4ef8 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -171,10 +171,11 @@ def _copy_common_includes(self): self._copy_or_merge_includes("modules", modules_yaml_path, modules_yaml_modulesys_path, destination) # Merge or copy common package config(s) packages_yaml_path = os.path.join(common_path, "packages.yaml") - if not "@" in self.compiler: - raise Exception("Invalid value for self.compiler, '@' not found. " + \ - "Expected: 'name@version' or 'name@=version'") - compiler_name, compiler_version = self.compiler.replace("@=", "@").split('@') + if "@" in self.compiler: + compiler_name, compiler_version = self.compiler.replace("@=", "@").split('@') + else: + compiler_name = self.compiler + compiler_version = None packages_compiler_yaml_path = os.path.join(common_path, f"packages_{compiler_name}.yaml") destination = os.path.join(env_common_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) @@ -194,7 +195,7 @@ def _copy_site_includes(self): site_name = "site" self.includes.append(site_name) - env_path = self.site_configs_dir() + site_path = self.site_configs_dir() env_site_dir = os.path.join(self.env_dir(), site_name) logging.info(f"Copying site includes from {self.site_configs_dir()} ...\n ... to {env_site_dir}") shutil.copytree( @@ -202,19 +203,15 @@ def _copy_site_includes(self): ) # Merge or copy site module config(s) lmod_or_tcl = self.get_lmod_or_tcl(self.site_configs_dir()) - modules_yaml_path = os.path.join(env_path, "modules.yaml") - modules_yaml_modulesys_path = os.path.join(env_path, f"modules_{lmod_or_tcl}.yaml") + modules_yaml_path = os.path.join(site_path, "modules.yaml") + modules_yaml_modulesys_path = os.path.join(site_path, f"modules_{lmod_or_tcl}.yaml") destination = os.path.join(env_site_dir, "modules.yaml") self._copy_or_merge_includes("modules", modules_yaml_path, modules_yaml_modulesys_path, destination) - # Merge or copy site package config(s) - packages_yaml_path = os.path.join(env_path, "packages.yaml") - if not "@" in self.compiler: - raise Exception("Invalid value for self.compiler, '@' not found. " + \ - "Expected: 'name@version' or 'name@=version'") - compiler_name, compiler_version = self.compiler.replace("@=", "@").split('@') - packages_compiler_yaml_path = os.path.join(env_path, f"packages_{compiler_name}-{compiler_version}.yaml") + # Merge or copy site package config(s), issue a warning if compiler-dependent package config doesn't exist + packages_yaml_path = os.path.join(site_path, "packages.yaml") + packages_compiler_yaml_path = os.path.join(site_path, f"packages_{self.compiler}.yaml") if not os.path.exists(packages_compiler_yaml_path): - raise Exception(f"File {packages_compiler_yaml_path} not found. ") + logging.warning(" {packages_compiler_yaml_path} not found, please check if this is correct") destination = os.path.join(env_site_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) @@ -357,15 +354,20 @@ def write(self): with open(os.path.join(env_repo_path, "repo.yaml"), "w") as f: f.write("repo:\n namespace: envrepo") repo_paths = spack.config.get("repos") - repo_paths = [p.replace("$spack/", spack.paths.spack_root + "/") for p in repo_paths] + repo_paths = [p.replace("${SPACK_STACK_DIR}", os.getenv("SPACK_STACK_DIR")) for p in repo_paths.values()] for pkg_name in self.modifypkg: pkg_found = False for repo_path in repo_paths: - pkg_path = os.path.join(repo_path, "packages", pkg_name) + # Replace dashes with underscores + pkg_subdir = pkg_name.replace("-","_") + # Prepend leading digits with underscores + pkg_subdir = '_' + pkg_subdir if pkg_subdir[0].isdigit() else pkg_subdir + pkg_path = os.path.join(repo_path, "packages", pkg_subdir) + # Copy package to env repo if os.path.exists(pkg_path): shutil.copytree( pkg_path, - os.path.join(env_pkgs_path, pkg_name), + os.path.join(env_pkgs_path, pkg_subdir), ignore=shutil.ignore_patterns("__pycache__"), dirs_exist_ok=True, ) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py index 3be5afa38..e3324edb6 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py @@ -1,3 +1,4 @@ +import shutil import os import pytest @@ -6,7 +7,7 @@ import spack.environment as ev import spack.main -stack_create = spack.main.SpackCommand("stack") +spack_stack_cmd = spack.main.SpackCommand("stack") # Find spack-stack directory assuming this Spack instance @@ -30,49 +31,65 @@ def test_setup_meta_modules(): return os.makedirs(test_dir, exist_ok=True) + env_root_dir = os.path.join(test_dir) - env_dir = os.path.join(test_dir) - stack_create("create", "env", "--dir", env_dir, "--name", "modtest", "--compiler", "gcc") + env_name = "modtest1" + env_dir = os.path.join(env_root_dir, env_name) + module_dir = os.path.join(env_dir, "install", "modulefiles") + if os.path.exists(env_dir): + shutil.rmtree(env_dir) + + spack_stack_cmd("create", "env", "--dir", env_root_dir, "--name", env_name, "--compiler", "gcc") # Create empty env - env_dir = os.path.join(env_dir, "modtest") env = ev.Environment(manifest_dir=env_dir) ev.activate(env) - comp = "gcc" - comp_ver = "9.4.0" - mpi = "openmpi" - mpi_ver = "4.0.1" - module_dir = os.path.join(env_dir, "install", "modulefiles") - compiler_module_path = os.path.join(module_dir, comp, comp_ver) - mpi_module_path = os.path.join(module_dir, mpi, mpi_ver, comp, comp_ver) - - # Setup env and pretend that a build exists - # by creating the module directory structure. - scope = env.scope_name - # Mock compiler config - compiler_def = { - "compiler": { - "flags": {}, - "modules": [], - "paths": {"cc": "gcc", "cxx": "g++", "f77": "gfortran", "fc": "gfortran"}, - "extra_rpaths": [], - "operating_system": "test", - "target": "test", - "environment": {}, - "spec": "gcc@9.4.0", - } - } - compiler_config = [compiler_def] - spack.config.set("compilers", compiler_config, scope=scope) - spack.config.add("packages:all:compiler:[{}]".format(comp), scope=scope) - spack.config.add("packages:all:providers:mpi:[{}]".format(mpi), scope=scope) - spack.config.add("packages:openmpi:version:[{}]".format(mpi_ver)) - spack.main.SpackCommand("stack") - os.makedirs(compiler_module_path) - os.makedirs(mpi_module_path) - cmd = spack.main.SpackCommand("external") - cmd("find", "python") - with env.write_transaction(): - env.write() - stack_create("setup-meta-modules") + packages_definition = """ +packages: + all: + prefer: + - '%gcc' + gcc: + externals: + - spec: gcc@13.2.1 languages:='c,c++,fortran' + prefix: /opt/rh/gcc-toolset-13/root/usr + extra_attributes: + compilers: + c: /opt/rh/gcc-toolset-13/root/usr/bin/gcc + cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ + fortran: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@13.2.1%gcc@13.2.1 + prefix: /opt/rh/gcc-toolset-13/root/usr + mpi: + buildable: false + openmpi: + externals: + - spec: openmpi@4.1.8 %gcc@13.2.1 + prefix: /opt/rh/gcc-toolset-13/root/usr + python: + buildable: false + externals: + - spec: python@3.11.11 + prefix: /opt/rh/gcc-toolset-13/root/usr +""" + site_packages_yaml = os.path.join(env_dir, "site", "packages.yaml") + if os.path.exists(site_packages_yaml): + raise Exception("Not implemented: appending to existing {site_packages_yaml}") + with open(site_packages_yaml, 'w') as f: + f.write(packages_definition) + + cmd = spack.main.SpackCommand("install") + cmd("--add", "--no-cache", "gcc", "openmpi", "python") + + spack_stack_cmd("setup-meta-modules") + + expected_comp_meta_module = os.path.join(module_dir, "Core", "stack-gcc", "13.2.1") + assert( + os.path.exists(expected_comp_meta_module), + f"Expected module {expected_comp_meta_module} not found" + ) + + expected_mpi_meta_module = os.path.join(module_dir, "gcc", "13.2.1", "stack-openmpi", "4.1.8") diff --git a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_stack_create.py b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_stack_create.py index 64846bda1..c8bf7489c 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_stack_create.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_stack_create.py @@ -1,5 +1,6 @@ -import shutil import os +import re +import shutil import pytest @@ -35,11 +36,18 @@ def all_templates(): def all_sites(): site_path = stack_path("configs", "sites") if site_path: + #sites = {} sites = [] _, tiers, _ = next(os.walk(site_path)) for tier in tiers: _, tier_sites, _ = next(os.walk(stack_path("configs", "sites", tier))) - sites += list(tier_sites) + for site in tier_sites: + _, _, files_in_site_dir = next(os.walk(stack_path("configs", "sites", tier, site))) + compilers = [] + for file in files_in_site_dir: + if file.startswith("packages_") and file.endswith(".yaml"): + compilers.append(file.replace("packages_", "").replace(".yaml", "")) + sites.append([site, compilers]) return sites else: return None @@ -91,18 +99,24 @@ def test_apps(template): def test_sites(site): if not site: return + name = site[0] + compilers = site[1] + # Assign dummy gcc compiler for sites without packages_*.yaml + if not compilers: + compilers = ["gcc"] if os.path.exists(test_dir): shutil.rmtree(test_dir) - stack_create( - "create", - "env", - "--site", - site, - "--dir", - test_dir, - "--compiler", - "gcc" - ) + for compiler in compilers: + stack_create( + "create", + "env", + "--site", + name, + "--dir", + test_dir, + "--compiler", + compiler + ) @pytest.mark.extension("stack") @@ -151,7 +165,7 @@ def test_modules(): @pytest.mark.extension("stack") @pytest.mark.filterwarnings("ignore::UserWarning") def test_compilers(): - for compiler in ["gcc", "oneapi"]: + for compiler in ["gcc-13.2.1", "oneapi-2025.1.0"]: if os.path.exists(test_dir): shutil.rmtree(test_dir) stack_create( @@ -166,14 +180,21 @@ def test_compilers(): "--compiler", compiler ) - packages_yaml_path = os.path.join(test_dir, "compiler_test", "site", "packages.yaml") - with open(packages_yaml_path, "r") as f: - packages_yaml_txt = f.read() - assert f"{compiler}@" in packages_yaml_txt - env_yaml_path = os.path.join(test_dir, "compiler_test", "spack.yaml") - with open(env_yaml_path, "r") as f: - env_yaml_txt = f.read() - assert f"%{compiler}" in env_yaml_txt + legacy_compiler_name, compiler_version = ( + lambda m: (compiler[:m.start()], compiler[m.start()+1:]) if m else (compiler, ''))(re.search(r'-(?=\d)', compiler) + ) + if legacy_compiler_name in spack.aliases.LEGACY_COMPILER_TO_BUILTIN.keys(): + builtin_compiler_name = spack.aliases.LEGACY_COMPILER_TO_BUILTIN[legacy_compiler_name] + else: + builtin_compiler_name = legacy_compiler_name + site_packages_yaml_path = os.path.join(test_dir, "compiler_test", "site", "packages.yaml") + with open(site_packages_yaml_path, "r") as f: + site_packages_yaml = f.read() + assert f"{builtin_compiler_name}@{compiler_version}" in site_packages_yaml + common_packages_yaml_path = os.path.join(test_dir, "compiler_test", "common", "packages.yaml") + with open(common_packages_yaml_path, "r") as f: + common_packages_yaml = f.read() + assert f"%{legacy_compiler_name}" in common_packages_yaml @pytest.mark.extension("stack") @@ -275,7 +296,7 @@ def test_modifypkg(): custom_hdf5_path ), "'--modify-pkg hdf5' failed to create custom package.py" custom_ufswm_path = os.path.join( - test_dir, "modifypkg_test/envrepo/packages/ufs-weather-model-env/package.py" + test_dir, "modifypkg_test/envrepo/packages/ufs_weather_model_env/package.py" ) assert os.path.exists( custom_ufswm_path @@ -288,14 +309,16 @@ def test_modifypkg(): hdf5_spack_path, "package.py" ), "Incorrect hdf5 location in modifypkg_test" ufswm_spack_path = spack_cmd( - "--env-dir", env_dir, "location", "--package-dir", "ufs-weather-model-env", output=str + "--env-dir", env_dir, "location", "--package-dir", "ufs_weather_model_env", output=str ).strip() assert custom_ufswm_path == os.path.join( ufswm_spack_path, "package.py" ), "Incorrect ufs-weather-model-env location in modifypkg_test" netcdfc_spack_path = spack_cmd( - "--env-dir", env_dir, "location", "--package-dir", "netcdf-c", output=str + "--env-dir", env_dir, "location", "--package-dir", "netcdf_c", output=str ).strip() + print("DH DEBUG 1:", os.path.join(spack.config.get("repos")['builtin'], "packages/netcdf_c")) + print("DH DEBUG 2:", netcdfc_spack_path) assert ( - os.path.join(spack.paths.packages_path, "packages/netcdf-c") == netcdfc_spack_path + os.path.join(spack.config.get("repos")['builtin'].replace("${SPACK_STACK_DIR}", os.getenv("SPACK_STACK_DIR")), "packages/netcdf_c") == netcdfc_spack_path ), "Incorrect netcdf-c location in modifypkg_test" From 5ffb61cbe8dc135f728a208f2fe6bde49b82cacc Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 Aug 2025 13:25:15 -0600 Subject: [PATCH 040/163] Remove leftover debugging print in spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py --- spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 4bdfecc2c..96a601ff1 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -304,8 +304,6 @@ def setup_meta_modules(): specs = spack.store.STORE.db.query(hashes=hashes) q = ProviderIndex(specs=specs, repository=spack.repo.PATH) - print(f"DH DEBUG: specs = {specs}") - c_providers = q.providers_for("c") cxx_providers = q.providers_for("cxx") fortran_providers = q.providers_for("fortran") From 6aac98f8ec15ba5388e40b203cbe949bc5c2e31b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 Aug 2025 13:40:27 -0600 Subject: [PATCH 041/163] Bug fixes for spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py - look for gcc in /usr --- .../jcsda-emc/spack-stack/stack/stack_env.py | 2 +- .../tests/test_setup_meta_modules.py | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index 8787d4ef8..ae9de7970 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -211,7 +211,7 @@ def _copy_site_includes(self): packages_yaml_path = os.path.join(site_path, "packages.yaml") packages_compiler_yaml_path = os.path.join(site_path, f"packages_{self.compiler}.yaml") if not os.path.exists(packages_compiler_yaml_path): - logging.warning(" {packages_compiler_yaml_path} not found, please check if this is correct") + logging.warning(f"\nWARNING: {packages_compiler_yaml_path} not found, please check if this is correct\n") destination = os.path.join(env_site_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py index e3324edb6..d72845017 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py @@ -52,28 +52,28 @@ def test_setup_meta_modules(): - '%gcc' gcc: externals: - - spec: gcc@13.2.1 languages:='c,c++,fortran' - prefix: /opt/rh/gcc-toolset-13/root/usr + - spec: gcc@11.5.0 languages:='c,c++,fortran' + prefix: /usr extra_attributes: compilers: - c: /opt/rh/gcc-toolset-13/root/usr/bin/gcc - cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ - fortran: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran gcc-runtime: externals: - - spec: gcc-runtime@13.2.1%gcc@13.2.1 - prefix: /opt/rh/gcc-toolset-13/root/usr + - spec: gcc-runtime@11.5.0 %gcc@11.5.0 + prefix: /usr mpi: buildable: false openmpi: externals: - - spec: openmpi@4.1.8 %gcc@13.2.1 - prefix: /opt/rh/gcc-toolset-13/root/usr + - spec: openmpi@4.1.8 %gcc@11.5.0 + prefix: /usr python: buildable: false externals: - spec: python@3.11.11 - prefix: /opt/rh/gcc-toolset-13/root/usr + prefix: /usr """ site_packages_yaml = os.path.join(env_dir, "site", "packages.yaml") if os.path.exists(site_packages_yaml): @@ -86,7 +86,7 @@ def test_setup_meta_modules(): spack_stack_cmd("setup-meta-modules") - expected_comp_meta_module = os.path.join(module_dir, "Core", "stack-gcc", "13.2.1") + expected_comp_meta_module = os.path.join(module_dir, "Core", "stack-gcc", "11.5.0") assert( os.path.exists(expected_comp_meta_module), f"Expected module {expected_comp_meta_module} not found" From 3bba9468551632e09f9ea71032edcb560e24d95c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 Aug 2025 12:49:51 -0700 Subject: [PATCH 042/163] Bug fix in spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py: core_core_compilers --> core_compilers --- spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 90351a102..879bf90c8 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -321,7 +321,7 @@ def setup_meta_modules(): # Core compilers is only a valid options for lmod if module_choice == "lmod": # Determine core compiler(s) and make sure they are not used - core_core_compilers = module_config["default"][module_choice]["core_compilers"] + core_compilers = module_config["default"][module_choice]["core_compilers"] logging.info(" ... core compilers: {}".format(core_compilers)) # Check that none of the compilers used for the stack is a core compiler for core_compiler in core_compilers: From 3aa1691c647952c8ca6b5f9842059d6bcfb60fba Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 18 Aug 2025 08:33:14 -0600 Subject: [PATCH 043/163] Add configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml --- ...yaml => packages_clang.yaml.notyetupdated} | 0 .../sites/tier1/atlantis/packages_intel.yaml | 20 ------ .../atlantis/packages_oneapi-2024.2.1.yaml | 71 +++++++++++++++++++ ...aml => packages_oneapi.yaml.notyetupdated} | 0 .../jcsda-emc/spack-stack/stack/stack_env.py | 21 ++++-- 5 files changed, 87 insertions(+), 25 deletions(-) rename configs/sites/tier1/atlantis/{packages_clang.yaml => packages_clang.yaml.notyetupdated} (100%) delete mode 100644 configs/sites/tier1/atlantis/packages_intel.yaml create mode 100644 configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml rename configs/sites/tier1/atlantis/{packages_oneapi.yaml => packages_oneapi.yaml.notyetupdated} (100%) diff --git a/configs/sites/tier1/atlantis/packages_clang.yaml b/configs/sites/tier1/atlantis/packages_clang.yaml.notyetupdated similarity index 100% rename from configs/sites/tier1/atlantis/packages_clang.yaml rename to configs/sites/tier1/atlantis/packages_clang.yaml.notyetupdated diff --git a/configs/sites/tier1/atlantis/packages_intel.yaml b/configs/sites/tier1/atlantis/packages_intel.yaml deleted file mode 100644 index e6eb565f3..000000000 --- a/configs/sites/tier1/atlantis/packages_intel.yaml +++ /dev/null @@ -1,20 +0,0 @@ -packages: - all: - compiler:: [intel@2021.6.0,gcc@11.2.0] - providers: - mpi:: [intel-oneapi-mpi@2021.6.0] - mpi: - buildable: False - intel-oneapi-mpi: - externals: - - spec: intel-oneapi-mpi@2021.6.0%intel@2021.6.0 +classic-names - prefix: /cm/shared/apps/intel/oneapi - modules: - - mpi/2021.6.0 - - slurm - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2022.1.0 - prefix: /cm/shared/apps/intel/oneapi - modules: - - mkl/2022.1.0 diff --git a/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml new file mode 100644 index 000000000..193656d56 --- /dev/null +++ b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml @@ -0,0 +1,71 @@ +packages: + mpi: + buildable: False + require: + - "intel-oneapi-mpi@2021.13" + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2024.2.1 + prefix: /cm/shared/apps/intel/oneapi-2024.2.1 + modules: + - intel-oneapi-2024.2.1 + - tbb/2021.13 + - compiler-rt/2024.2.1 + - compiler/2024.2.1 + extra_attributes: + compilers: + c: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/bin/icx + cxx: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/bin/icpx + fortran: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/bin/ifx + environment: + prepend_path: + PATH: /cm/local/apps/gcc/11.2.0/bin + CPATH: /cm/local/apps/gcc/11.2.0/include + extra_rpaths: + - /cm/local/apps/gcc/11.2.0/lib64 + - /cm/local/apps/gcc/11.2.0/lib + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 + prefix: /cm/shared/apps/intel/oneapi-2024.2.1 + modules: + - mpi/2021.13 + - slurm + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2 + prefix: /cm/shared/apps/intel/oneapi-2024.2.1 + modules: + - mkl/2024.2 + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2021.13 + prefix: /cm/shared/apps/intel/oneapi-2024.2.1 + modules: + - tbb/2021.13 + intel-oneapi-runtime: + # LINK TO BUG REPORT HERE + #buildable: false + externals: + - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 + prefix: /cm/shared/apps/intel/oneapi-2024.2.1 + modules: + - intel-oneapi-2024.2.1 + - tbb/2021.13 + - compiler-rt/2024.2.1 + gcc: + externals: + - spec: gcc@11.2.0 languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /cm/local/apps/gcc/11.2.0/gcc + cxx: /cm/local/apps/gcc/11.2.0/g++ + fortran: /cm/local/apps/gcc/11.2.0/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@11.2.0%gcc@11.2.0 + prefix: /cm/local/apps/gcc/11.2.0 diff --git a/configs/sites/tier1/atlantis/packages_oneapi.yaml b/configs/sites/tier1/atlantis/packages_oneapi.yaml.notyetupdated similarity index 100% rename from configs/sites/tier1/atlantis/packages_oneapi.yaml rename to configs/sites/tier1/atlantis/packages_oneapi.yaml.notyetupdated diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index ae9de7970..e3dc258d6 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -49,6 +49,19 @@ def spack_hash(): return get_git_revision_short_hash(spack.paths.spack_root) +def compiler_name_and_version_from_string(compiler): + """Return compiler name and version from a string that + contains name and version separated by - (as opposed + to @), e.g. gcc-13.3.1. If not version is provided, + return the name and "None" for the version.""" + compiler_name, compiler_version = ( + lambda m: (compiler[:m.start()], compiler[m.start()+1:]) if m else (compiler, ''))(re.search(r'-(?=\d)', compiler) + ) + if not compiler_version: + compiler_version = None + return (compiler_name, compiler_version) + + class StackEnv(object): """Represents a spack.yaml environment based on different configurations of sites and specs. Uses the Spack library @@ -171,12 +184,10 @@ def _copy_common_includes(self): self._copy_or_merge_includes("modules", modules_yaml_path, modules_yaml_modulesys_path, destination) # Merge or copy common package config(s) packages_yaml_path = os.path.join(common_path, "packages.yaml") - if "@" in self.compiler: - compiler_name, compiler_version = self.compiler.replace("@=", "@").split('@') - else: - compiler_name = self.compiler - compiler_version = None + (compiler_name, _) = compiler_name_and_version_from_string(self.compiler) packages_compiler_yaml_path = os.path.join(common_path, f"packages_{compiler_name}.yaml") + if not os.path.exists(packages_compiler_yaml_path): + logging.warning(f"\nWARNING: {packages_compiler_yaml_path} not found, please check if this is correct\n") destination = os.path.join(env_common_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) From 5371a28085d3308d23b4c34899f01d4221a12632 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 18 Aug 2025 09:30:24 -0600 Subject: [PATCH 044/163] Fix configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml --- .../sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml index 193656d56..839d0144a 100644 --- a/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml +++ b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml @@ -59,12 +59,12 @@ packages: gcc: externals: - spec: gcc@11.2.0 languages:='c,c++' - prefix: /usr + prefix: /cm/local/apps/gcc/11.2.0 extra_attributes: compilers: - c: /cm/local/apps/gcc/11.2.0/gcc - cxx: /cm/local/apps/gcc/11.2.0/g++ - fortran: /cm/local/apps/gcc/11.2.0/gfortran + c: /cm/local/apps/gcc/11.2.0/bin/gcc + cxx: /cm/local/apps/gcc/11.2.0/bin/g++ + fortran: /cm/local/apps/gcc/11.2.0/bin/gfortran gcc-runtime: externals: - spec: gcc-runtime@11.2.0%gcc@11.2.0 From 88aa521ae4f9867d1ab5dc70cf6fbb2049321fba Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Aug 2025 07:00:09 -0600 Subject: [PATCH 045/163] Re-enable GitHub action workflow ubuntu-ci-x86_64-gnu.yaml; remove Fortran from Intel oneAPI's backup GCC compiler for Bounty and Blackpearl --- .github/workflows/{TMP_OFF => }/ubuntu-ci-x86_64-gnu.yaml | 0 configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml | 2 +- configs/sites/tier2/blackpearl/packages_oneapi-2025.1.0.yaml | 2 +- configs/sites/tier2/bounty/packages_oneapi-2025.1.1.yaml | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename .github/workflows/{TMP_OFF => }/ubuntu-ci-x86_64-gnu.yaml (100%) diff --git a/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml similarity index 100% rename from .github/workflows/TMP_OFF/ubuntu-ci-x86_64-gnu.yaml rename to .github/workflows/ubuntu-ci-x86_64-gnu.yaml diff --git a/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml b/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml index 6f0e5a23a..e4e3fbc83 100644 --- a/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml +++ b/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml @@ -30,7 +30,7 @@ packages: prefix: /opt/intel/oneapi gcc: externals: - - spec: gcc@11.4.1 languages:='c,c++,fortran' + - spec: gcc@11.4.1 languages:='c,c++' prefix: /usr extra_attributes: compilers: diff --git a/configs/sites/tier2/blackpearl/packages_oneapi-2025.1.0.yaml b/configs/sites/tier2/blackpearl/packages_oneapi-2025.1.0.yaml index ab3e0217d..90445d17f 100644 --- a/configs/sites/tier2/blackpearl/packages_oneapi-2025.1.0.yaml +++ b/configs/sites/tier2/blackpearl/packages_oneapi-2025.1.0.yaml @@ -30,7 +30,7 @@ packages: prefix: /opt/intel/oneapi gcc: externals: - - spec: gcc@11.4.1 languages:='c,c++,fortran' + - spec: gcc@11.4.1 languages:='c,c++' prefix: /usr extra_attributes: compilers: diff --git a/configs/sites/tier2/bounty/packages_oneapi-2025.1.1.yaml b/configs/sites/tier2/bounty/packages_oneapi-2025.1.1.yaml index bf598e38e..1d70245aa 100644 --- a/configs/sites/tier2/bounty/packages_oneapi-2025.1.1.yaml +++ b/configs/sites/tier2/bounty/packages_oneapi-2025.1.1.yaml @@ -30,7 +30,7 @@ packages: prefix: /opt/intel/oneapi gcc: externals: - - spec: gcc@11.5.0 languages:='c,c++,fortran' + - spec: gcc@11.5.0 languages:='c,c++' prefix: /usr extra_attributes: compilers: From 328dcdd3f617edfc9ffb09331597d7bef8fb6df7 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Aug 2025 07:07:07 -0600 Subject: [PATCH 046/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 8faec0553..ee829f7fd 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 8faec05536c7bae79c33c1ae5ba5404631dfbef5 +Subproject commit ee829f7fd4bdb3a6edb61412996f66fff78aab0b From 083bb039b3dcecf8740d8e6d4a2230f5a410e092 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Aug 2025 07:25:51 -0600 Subject: [PATCH 047/163] Update configs/common/packages.yaml and configs/sites/tier1/packages.yaml to reduce number of duplicate packages for unified environment with oneAPI --- configs/common/packages.yaml | 12 ++++++------ configs/sites/tier1/atlantis/packages.yaml | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 440abd19c..bd17a0a8f 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -230,10 +230,10 @@ packages: # ncview - check Orion and Discover site configs nemsio: require: - - '@2.5.4' + - '@2.5.5' nemsiogfs: require: - - '@2.5.3' + - '@2.5.4' nccmp: require: - '@1.9.0.1' @@ -347,9 +347,9 @@ packages: py-python-dateutil: require: - '@2' - #py-ruamel-yaml: - # require: - # - '@0.17.16' + py-ruamel-yaml: + require: + - '@0.17.16' # Pin the py-setuptools version to avoid duplicate Python packages py-setuptools: require: @@ -410,7 +410,7 @@ packages: - '@3.6.0' wrf-io: require: - - '@1.2.0' + - '@1.3.0' zstd: require: - '@1.5.7' diff --git a/configs/sites/tier1/atlantis/packages.yaml b/configs/sites/tier1/atlantis/packages.yaml index eef70a0e9..bd98ec047 100644 --- a/configs/sites/tier1/atlantis/packages.yaml +++ b/configs/sites/tier1/atlantis/packages.yaml @@ -42,10 +42,12 @@ packages: - spec: gawk@4.2.1 prefix: /usr git: + buildable: False externals: - spec: git@2.43.5+tcltk prefix: /usr git-lfs: + buildable: False externals: - spec: git-lfs@3.4.1 prefix: /usr From dd7e72ed7402ff20828517c5fd101d7c4f470033 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Aug 2025 07:26:40 -0600 Subject: [PATCH 048/163] .github/workflows/ubuntu-ci-x86_64-gnu.yaml: test only templates neptune-dev and cylc-dev --- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 5d21c35fa..85a892f1d 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -51,7 +51,8 @@ jobs: # Set up spack-stack source ./setup.sh - declare -a TEMPLATES=("unified-dev" "skylab-dev" "cylc-dev" "neptune-dev") + #declare -a TEMPLATES=("unified-dev" "skylab-dev" "cylc-dev" "neptune-dev") + declare -a TEMPLATES=("cylc-dev" "neptune-dev") for TEMPLATE in "${TEMPLATES[@]}"; do if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then export ENVNAME=ue-gcc-11.4.0-buildcache @@ -106,7 +107,7 @@ jobs: # Concretize and check for duplicates spack concretize 2>&1 | tee log.concretize.gnu-11.4.0-buildcache - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i neptune-env + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env # Add and update source cache spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/ @@ -177,7 +178,7 @@ jobs: # Concretize and check for duplicates spack concretize --force 2>&1 | tee log.concretize.gnu-11.4.0 - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env # Add binary cache back in spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ From eeaf18d63113409f2a1c296840466cc3c737bf97 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Aug 2025 14:05:35 -0600 Subject: [PATCH 049/163] Update configs/sites/tier2/bounty/packages.yaml: git and git-lfs are buildable: false --- configs/sites/tier2/bounty/packages.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/configs/sites/tier2/bounty/packages.yaml b/configs/sites/tier2/bounty/packages.yaml index 2cdf9dbbe..a1b4ef1f9 100644 --- a/configs/sites/tier2/bounty/packages.yaml +++ b/configs/sites/tier2/bounty/packages.yaml @@ -46,10 +46,12 @@ packages: - spec: gettext@0.21 prefix: /usr git: + buildable: False externals: - spec: git@2.43.5~tcltk prefix: /usr git-lfs: + buildable: False externals: - spec: git-lfs@3.4.1 prefix: /usr @@ -83,7 +85,8 @@ packages: - spec: pkgconf@1.7.3 prefix: /usr qt: - buildable: False + # Yet another bug in spack v1 ... grrr ... + #buildable: False externals: - spec: qt@5.15.9 prefix: /usr/lib64/qt5 From c9d7a16dfb468c0d283428b997fb41e835151b22 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Aug 2025 14:06:03 -0600 Subject: [PATCH 050/163] More updates to configs/common/packages.yaml to remove duplicates when concretizing ue-oneapi --- configs/common/packages.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index bd17a0a8f..9215184bf 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -347,6 +347,11 @@ packages: py-python-dateutil: require: - '@2' + # To avoid duplicate packages + py-requests: + require: + - '@2.32.3' + # To avoid duplicate packages py-ruamel-yaml: require: - '@0.17.16' @@ -358,10 +363,10 @@ packages: py-torch: require: - +custom-protobuf ~mkldnn - ## To avoid duplicate packages - #py-urllib3: - # require: - # - '@1.26.12' + # To avoid duplicate packages + py-urllib3: + require: + - '@1.26.12' ## To avoid duplicate packages #py-versioneer: # require: From f186a0d24777a1e498b75f6fe798c4ce29b84936 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Aug 2025 14:17:44 -0600 Subject: [PATCH 051/163] Add esmf@8.9.0 --- configs/common/modules_lmod.yaml | 10 ++-------- configs/common/modules_tcl.yaml | 10 ++-------- configs/common/packages.yaml | 2 +- configs/templates/neptune-dev/spack.yaml | 8 ++++---- configs/templates/skylab-dev/spack.yaml | 8 ++++---- configs/templates/unified-dev/spack.yaml | 8 ++++---- repos/builtin | 2 +- 7 files changed, 18 insertions(+), 30 deletions(-) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 410275773..87b633b78 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -136,14 +136,8 @@ modules: ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug' ^esmf@8.8.0~debug snapshot=none: 'esmf-8.8.0' ^esmf@8.8.0+debug snapshot=none: 'esmf-8.8.0-debug' - ^esmf@8.9.0b08~debug snapshot=b08: 'esmf-8.9.0b08' - ^esmf@8.9.0b08+debug snapshot=b08: 'esmf-8.9.0b08-debug' - ^esmf@8.9.0b09~debug snapshot=b09: 'esmf-8.9.0b09' - ^esmf@8.9.0b09+debug snapshot=b09: 'esmf-8.9.0b09-debug' - ^esmf@8.9.0b11~debug snapshot=b11: 'esmf-8.9.0b11' - ^esmf@8.9.0b11+debug snapshot=b11: 'esmf-8.9.0b11-debug' - ^esmf@8.9.0b12~debug snapshot=b12: 'esmf-8.9.0b12' - ^esmf@8.9.0b12+debug snapshot=b12: 'esmf-8.9.0b12-debug' + ^esmf@8.9.0~debug snapshot=none: 'esmf-8.9.0' + ^esmf@8.9.0+debug snapshot=none: 'esmf-8.9.0-debug' openmpi: environment: set: diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 3df12ea1f..74186d8de 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -134,14 +134,8 @@ modules: ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug' ^esmf@8.8.0~debug snapshot=none: 'esmf-8.8.0' ^esmf@8.8.0+debug snapshot=none: 'esmf-8.8.0-debug' - ^esmf@8.9.0b08~debug snapshot=b08: 'esmf-8.9.0b08' - ^esmf@8.9.0b08+debug snapshot=b08: 'esmf-8.9.0b08-debug' - ^esmf@8.9.0b09~debug snapshot=b09: 'esmf-8.9.0b09' - ^esmf@8.9.0b09+debug snapshot=b09: 'esmf-8.9.0b09-debug' - ^esmf@8.9.0b11~debug snapshot=b11: 'esmf-8.9.0b11' - ^esmf@8.9.0b11+debug snapshot=b11: 'esmf-8.9.0b11-debug' - ^esmf@8.9.0b12~debug snapshot=b12: 'esmf-8.9.0b12' - ^esmf@8.9.0b12+debug snapshot=b12: 'esmf-8.9.0b12-debug' + ^esmf@8.9.0~debug snapshot=none: 'esmf-8.9.0' + ^esmf@8.9.0+debug snapshot=none: 'esmf-8.9.0-debug' openmpi: environment: set: diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 837afee4c..7d30bf05d 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -84,7 +84,7 @@ packages: esmf: require: - ~xerces ~pnetcdf +shared +external-parallelio +python - - any_of: ['@=8.6.1 snapshot=none', '@=8.8.0 snapshot=none', '@=8.9.0b12 snapshot=b12'] + - any_of: ['@=8.6.1 snapshot=none', '@=8.8.0 snapshot=none', '@=8.9.0 snapshot=none'] - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] when: '%intel' message: Extra ESMF compile options for Intel diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 6e71e3e66..22e1b1454 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -6,10 +6,10 @@ spack: include: [] specs: - - neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 - - neptune-env +debug +espc ^esmf@=8.9.0b12 snapshot=b12 - - neptune-python-env +xnrl ^neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 - - jedi-neptune-env ^neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 + - neptune-env ~debug +espc ^esmf@=8.9.0 + - neptune-env +debug +espc ^esmf@=8.9.0 + - neptune-python-env +xnrl ^neptune-env ~debug +espc ^esmf@=8.9.0 + - jedi-neptune-env ^neptune-env ~debug +espc ^esmf@=8.9.0 - crtm@3.1.2 - crtm@v2.4.1-jedi.2 diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 308326304..7606f40e0 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -14,12 +14,12 @@ spack: - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.9.0b12 + - jedi-neptune-env ^esmf@=8.9.0 - jedi-tools-env - jedi-ufs-env ^esmf@=8.8.0 - jedi-um-env - - neptune-env ^esmf@=8.9.0b12 - - neptune-python-env ^esmf@=8.9.0b12 + - neptune-env ^esmf@=8.9.0 + - neptune-python-env ^esmf@=8.9.0 - soca-env # Various crtm tags (list all to avoid duplicate packages) @@ -34,7 +34,7 @@ spack: # Various esmf tags (list all to avoid duplicate packages) - esmf@=8.6.1 snapshot=none - esmf@=8.8.0 snapshot=none - - esmf@=8.9.0b12 snapshot=b12 + - esmf@=8.9.0 specs: - matrix: diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 335f7c835..1b1941df2 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -15,12 +15,12 @@ spack: - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.9.0b12 + - jedi-neptune-env ^esmf@=8.9.0 #- jedi-tools-env - jedi-ufs-env ^esmf@=8.8.0 - jedi-um-env - - neptune-env ^esmf@=8.9.0b12 - - neptune-python-env ^esmf@=8.9.0b12 + - neptune-env ^esmf@=8.9.0 + - neptune-python-env ^esmf@=8.9.0 - soca-env - ufs-srw-app-env ^esmf@=8.8.0 ^crtm@=3.1.2 - ufs-weather-model-env ^esmf@=8.8.0 ^crtm@=3.1.2 @@ -37,7 +37,7 @@ spack: # Various esmf tags (list all to avoid duplicate packages) - esmf@=8.6.1 snapshot=none - esmf@=8.8.0 snapshot=none - - esmf@=8.9.0b12 snapshot=b12 + - esmf@=8.9.0 # MADIS for WCOSS2 decoders. - madis@4.5 diff --git a/repos/builtin b/repos/builtin index ee829f7fd..822e3c13c 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit ee829f7fd4bdb3a6edb61412996f66fff78aab0b +Subproject commit 822e3c13cb681de1d8c70c607e950ac5547cfde4 From e59406422831f4e79c711010a748606e63dd245d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Aug 2025 14:17:44 -0600 Subject: [PATCH 052/163] Add esmf@8.9.0 --- configs/common/modules_lmod.yaml | 10 ++-------- configs/common/modules_tcl.yaml | 10 ++-------- configs/common/packages.yaml | 2 +- configs/templates/neptune-dev/spack.yaml | 8 ++++---- configs/templates/skylab-dev/spack.yaml | 12 ++++++------ configs/templates/unified-dev/spack.yaml | 12 ++++++------ repos/builtin | 2 +- 7 files changed, 22 insertions(+), 34 deletions(-) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 410275773..87b633b78 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -136,14 +136,8 @@ modules: ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug' ^esmf@8.8.0~debug snapshot=none: 'esmf-8.8.0' ^esmf@8.8.0+debug snapshot=none: 'esmf-8.8.0-debug' - ^esmf@8.9.0b08~debug snapshot=b08: 'esmf-8.9.0b08' - ^esmf@8.9.0b08+debug snapshot=b08: 'esmf-8.9.0b08-debug' - ^esmf@8.9.0b09~debug snapshot=b09: 'esmf-8.9.0b09' - ^esmf@8.9.0b09+debug snapshot=b09: 'esmf-8.9.0b09-debug' - ^esmf@8.9.0b11~debug snapshot=b11: 'esmf-8.9.0b11' - ^esmf@8.9.0b11+debug snapshot=b11: 'esmf-8.9.0b11-debug' - ^esmf@8.9.0b12~debug snapshot=b12: 'esmf-8.9.0b12' - ^esmf@8.9.0b12+debug snapshot=b12: 'esmf-8.9.0b12-debug' + ^esmf@8.9.0~debug snapshot=none: 'esmf-8.9.0' + ^esmf@8.9.0+debug snapshot=none: 'esmf-8.9.0-debug' openmpi: environment: set: diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 3df12ea1f..74186d8de 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -134,14 +134,8 @@ modules: ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug' ^esmf@8.8.0~debug snapshot=none: 'esmf-8.8.0' ^esmf@8.8.0+debug snapshot=none: 'esmf-8.8.0-debug' - ^esmf@8.9.0b08~debug snapshot=b08: 'esmf-8.9.0b08' - ^esmf@8.9.0b08+debug snapshot=b08: 'esmf-8.9.0b08-debug' - ^esmf@8.9.0b09~debug snapshot=b09: 'esmf-8.9.0b09' - ^esmf@8.9.0b09+debug snapshot=b09: 'esmf-8.9.0b09-debug' - ^esmf@8.9.0b11~debug snapshot=b11: 'esmf-8.9.0b11' - ^esmf@8.9.0b11+debug snapshot=b11: 'esmf-8.9.0b11-debug' - ^esmf@8.9.0b12~debug snapshot=b12: 'esmf-8.9.0b12' - ^esmf@8.9.0b12+debug snapshot=b12: 'esmf-8.9.0b12-debug' + ^esmf@8.9.0~debug snapshot=none: 'esmf-8.9.0' + ^esmf@8.9.0+debug snapshot=none: 'esmf-8.9.0-debug' openmpi: environment: set: diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 837afee4c..7d30bf05d 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -84,7 +84,7 @@ packages: esmf: require: - ~xerces ~pnetcdf +shared +external-parallelio +python - - any_of: ['@=8.6.1 snapshot=none', '@=8.8.0 snapshot=none', '@=8.9.0b12 snapshot=b12'] + - any_of: ['@=8.6.1 snapshot=none', '@=8.8.0 snapshot=none', '@=8.9.0 snapshot=none'] - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] when: '%intel' message: Extra ESMF compile options for Intel diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 6e71e3e66..22e1b1454 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -6,10 +6,10 @@ spack: include: [] specs: - - neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 - - neptune-env +debug +espc ^esmf@=8.9.0b12 snapshot=b12 - - neptune-python-env +xnrl ^neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 - - jedi-neptune-env ^neptune-env ~debug +espc ^esmf@=8.9.0b12 snapshot=b12 + - neptune-env ~debug +espc ^esmf@=8.9.0 + - neptune-env +debug +espc ^esmf@=8.9.0 + - neptune-python-env +xnrl ^neptune-env ~debug +espc ^esmf@=8.9.0 + - jedi-neptune-env ^neptune-env ~debug +espc ^esmf@=8.9.0 - crtm@3.1.2 - crtm@v2.4.1-jedi.2 diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 308326304..560c2900a 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -14,12 +14,12 @@ spack: - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.9.0b12 + - jedi-neptune-env ^esmf@=8.9.0 - jedi-tools-env - jedi-ufs-env ^esmf@=8.8.0 - jedi-um-env - - neptune-env ^esmf@=8.9.0b12 - - neptune-python-env ^esmf@=8.9.0b12 + - neptune-env ^esmf@=8.9.0 + - neptune-python-env ^esmf@=8.9.0 - soca-env # Various crtm tags (list all to avoid duplicate packages) @@ -32,9 +32,9 @@ spack: - mapl@2.53.4 ^esmf@8.8.0 # Various esmf tags (list all to avoid duplicate packages) - - esmf@=8.6.1 snapshot=none - - esmf@=8.8.0 snapshot=none - - esmf@=8.9.0b12 snapshot=b12 + - esmf@=8.6.1 + - esmf@=8.8.0 + - esmf@=8.9.0 specs: - matrix: diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 335f7c835..fc0db0b61 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -15,12 +15,12 @@ spack: - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.9.0b12 + - jedi-neptune-env ^esmf@=8.9.0 #- jedi-tools-env - jedi-ufs-env ^esmf@=8.8.0 - jedi-um-env - - neptune-env ^esmf@=8.9.0b12 - - neptune-python-env ^esmf@=8.9.0b12 + - neptune-env ^esmf@=8.9.0 + - neptune-python-env ^esmf@=8.9.0 - soca-env - ufs-srw-app-env ^esmf@=8.8.0 ^crtm@=3.1.2 - ufs-weather-model-env ^esmf@=8.8.0 ^crtm@=3.1.2 @@ -35,9 +35,9 @@ spack: - mapl@2.53.4 ^esmf@8.8.0 # Various esmf tags (list all to avoid duplicate packages) - - esmf@=8.6.1 snapshot=none - - esmf@=8.8.0 snapshot=none - - esmf@=8.9.0b12 snapshot=b12 + - esmf@=8.6.1 + - esmf@=8.8.0 + - esmf@=8.9.0 # MADIS for WCOSS2 decoders. - madis@4.5 diff --git a/repos/builtin b/repos/builtin index ee829f7fd..822e3c13c 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit ee829f7fd4bdb3a6edb61412996f66fff78aab0b +Subproject commit 822e3c13cb681de1d8c70c607e950ac5547cfde4 From 6e22e5706eeaff8844f933635e8e8f384304e472 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 Aug 2025 05:48:30 -0600 Subject: [PATCH 053/163] Bug fix in repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py --- .../spack_repo/spack_stack/packages/neptune_env/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py index a6121fb56..b2120fc79 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_env/package.py @@ -34,8 +34,8 @@ class NeptuneEnv(BundlePackage): depends_on("p4est", type="run") depends_on("w3emc", type="run") depends_on("ip", type="run") - depends_on("esmf@8.9.0b12 ~debug snapshot=b12", type="run", when="~debug") - depends_on("esmf@8.9.0b12 +debug snapshot=b12", type="run", when="+debug") + depends_on("esmf ~debug", type="run", when="~debug") + depends_on("esmf +debug", type="run", when="+debug") depends_on("nco", type="run") depends_on("mct", type="run") From d5a777df540229a96faa31415c4d766a325d5e3f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 Aug 2025 08:37:45 -0600 Subject: [PATCH 054/163] configs/common/packages.yaml: require py-beautifulsoup4@4.12.3 --- configs/common/packages.yaml | 7 ++ configs/sites/tier2/bounty/compilers.yaml.old | 71 ------------------- ..._clang.yaml => packages_clang-20.1.5.yaml} | 0 repos/builtin | 2 +- 4 files changed, 8 insertions(+), 72 deletions(-) delete mode 100644 configs/sites/tier2/bounty/compilers.yaml.old rename configs/sites/tier2/bounty/{packages_clang.yaml => packages_clang-20.1.5.yaml} (100%) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 7d30bf05d..6cf7ab800 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -292,6 +292,13 @@ packages: require: - '@3.11' - ~crypt + # Not sure about the exact version requirements, but earlier + # versions (definitely 4.5.3 and earlier) don't work with newer + # py-setuptools (version 68 and later) because of a breaking + # change: error in beautifulsoup4 setup command: use_2to3 is invalid + py-beautifulsoup4: + require: + - '@4.12.3:' py-cartopy: require: - +plotting diff --git a/configs/sites/tier2/bounty/compilers.yaml.old b/configs/sites/tier2/bounty/compilers.yaml.old deleted file mode 100644 index d0062218b..000000000 --- a/configs/sites/tier2/bounty/compilers.yaml.old +++ /dev/null @@ -1,71 +0,0 @@ -compilers: -- compiler: - spec: gcc@=11.5.0 - paths: - cc: /usr/bin/gcc - cxx: /usr/bin/g++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - flags: {} - operating_system: almalinux9 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@=13.3.1 - paths: - cc: /opt/rh/gcc-toolset-13/root/usr/bin/gcc - cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ - f77: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran - fc: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran - flags: {} - operating_system: almalinux9 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: oneapi@=2025.1.1 - paths: - cc: /opt/intel/oneapi/compiler/2025.1/bin/icx - cxx: /opt/intel/oneapi/compiler/2025.1/bin/icpx - f77: /opt/intel/oneapi/compiler/2025.1/bin/ifx - fc: /opt/intel/oneapi/compiler/2025.1/bin/ifx - flags: {} - operating_system: almalinux9 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@=20.1.5 - paths: - cc: /home/dom/prod/llvm-20.1.5/bin/clang - cxx: /home/dom/prod/llvm-20.1.5/bin/clang++ - f77: /home/dom/prod/llvm-20.1.5/bin/flang-new - fc: /home/dom/prod/llvm-20.1.5/bin/flang-new - flags: {} - operating_system: almalinux9 - target: x86_64 - modules: [] - environment: - append_path: - LD_LIBRARY_PATH: /home/dom/prod/llvm-20.1.5/lib - extra_rpaths: [/home/dom/prod/llvm-20.1.5/lib] -#- compiler: -# spec: aocc@=5.0.0 -# paths: -# cc: /home/dom/prod/aocc-5.0.0/bin/clang -# cxx: /home/dom/prod/aocc-5.0.0/bin/clang++ -# f77: /home/dom/prod/aocc-5.0.0/bin/flang -# fc: /home/dom/prod/aocc-5.0.0/bin/flang -# flags: {} -# operating_system: almalinux9 -# target: x86_64 -# modules: [] -# environment: {} -# #append_path: -# # For libquadmath etc -# # NEEDED? LD_LIBRARY_PATH: /usr/lib/gcc/x86_64-redhat-linux/11 -# extra_rpaths: [] diff --git a/configs/sites/tier2/bounty/packages_clang.yaml b/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml similarity index 100% rename from configs/sites/tier2/bounty/packages_clang.yaml rename to configs/sites/tier2/bounty/packages_clang-20.1.5.yaml diff --git a/repos/builtin b/repos/builtin index 822e3c13c..04a23bb01 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 822e3c13cb681de1d8c70c607e950ac5547cfde4 +Subproject commit 04a23bb01a02e775a2a9b7863fe34487f9be25b0 From 8d886f1d6353f8b97c23c731c359a89114e59508 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 Aug 2025 08:38:28 -0600 Subject: [PATCH 055/163] Update bounty site config: clang, newer openmpi --- configs/common/packages_clang.yaml | 33 +++++++++---------- .../tier2/bounty/packages_clang-20.1.5.yaml | 16 ++++++--- .../tier2/bounty/packages_gcc-13.3.1.yaml | 2 +- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/configs/common/packages_clang.yaml b/configs/common/packages_clang.yaml index 9fa764bc2..28baa48af 100644 --- a/configs/common/packages_clang.yaml +++ b/configs/common/packages_clang.yaml @@ -1,18 +1,17 @@ packages: - all: - providers: - blas: [openblas] - fftw-api: [fftw] - lapack: [openblas] - ectrans: - require: - - '~mkl +fftw' - gsibec: - require: - - '~mkl' - ip: - require: - - '~openmp' - py-numpy: - require: - - '^openblas' + all: + prefer: + - '%clang' + providers: + blas: [openblas] + fftw-api: [fftw] + lapack: [openblas] + ectrans: + require: + - ~mkl +fftw + ip: + require: + - '^openblas' + py-numpy: + require: + - '^openblas' diff --git a/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml b/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml index cd8a21c02..25d013f79 100644 --- a/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml +++ b/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml @@ -1,5 +1,13 @@ packages: - all: - compiler:: [clang@20.1.5] - providers: - mpi:: [openmpi@5.0.6] + mpi: + require: + - "openmpi@=5.0.8" + llvm: + externals: + - spec: llvm@20.1.5 +flang ~libomptarget + prefix: /home/dom/prod/llvm-20.1.5 + extra_attributes: + compilers: + c: /home/dom/prod/llvm-20.1.5/bin/clang + cxx: /home/dom/prod/llvm-20.1.5/bin/clang++ + fortran: /home/dom/prod/llvm-20.1.5/bin/flang-new diff --git a/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml b/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml index 05ca18a79..514689ece 100644 --- a/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml +++ b/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml @@ -1,7 +1,7 @@ packages: mpi: require: - - "openmpi@=5.0.6" + - "openmpi@=5.0.8" gcc: externals: - spec: gcc@13.3.1 languages:='c,c++,fortran' From 53649facc909859701f9bddefad2f22fe1b6bb82 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 Aug 2025 08:40:24 -0600 Subject: [PATCH 056/163] New env syntax to require certain mpi and prefer certain compiler in .github/workflows/ubuntu-ci-x86_64-gnu.yaml --- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 85a892f1d..7840739a4 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -94,8 +94,8 @@ jobs: spack config add config:install_tree:padded_length:200 # Set compiler and MPI specs - spack config add "packages:all:providers:mpi:[openmpi@5.0.3]" - spack config add "packages:all:compiler:[gcc@11.4.0]" + spack config add "packages:mpi:require:['openmpi@5.0.8']" + spack config add "packages:all:prefer:['%gcc']" # Add additional variants for MET packages, different from config/common/packages.yaml spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" From 2f0f55e10298b2a1bd4506e51cc0a50d123502dd Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 Aug 2025 14:17:55 -0600 Subject: [PATCH 057/163] [skip ci] Update configs/sites/tier1/navy-aws/packages.yaml and add configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml --- configs/sites/tier1/navy-aws/packages.yaml | 173 ++++++++++++++---- .../navy-aws/packages_oneapi-2025.2.1.yaml | 65 +++++++ 2 files changed, 201 insertions(+), 37 deletions(-) create mode 100644 configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml diff --git a/configs/sites/tier1/navy-aws/packages.yaml b/configs/sites/tier1/navy-aws/packages.yaml index ba08841e1..29e41fd75 100644 --- a/configs/sites/tier1/navy-aws/packages.yaml +++ b/configs/sites/tier1/navy-aws/packages.yaml @@ -1,21 +1,33 @@ packages: + # Modification of common packages + # Prevent building zlib, zlib-ng + zlib-api: + buildable: false + + # All other packages listed alphabetically autoconf: externals: - spec: autoconf@2.69 prefix: /usr automake: externals: - - spec: automake@1.16.1 + - spec: automake@1.16.2 + prefix: /usr + bash: + externals: + - spec: bash@5.1.8 prefix: /usr binutils: externals: - - spec: binutils@2.40.21~gold~headers - prefix: /opt/rh/gcc-toolset-13/root/usr - - spec: binutils@2.30.123~gold+headers + - spec: binutils@2.35.2~gold~headers prefix: /usr bison: externals: - - spec: bison@3.0.4 + - spec: bison@3.7.4 + prefix: /usr + bzip2: + externals: + - spec: bzip2@1.0.8 prefix: /usr cmake: externals: @@ -23,104 +35,191 @@ packages: prefix: /usr coreutils: externals: - - spec: coreutils@8.30 + - spec: coreutils@8.32 + prefix: /usr + cpio: + externals: + - spec: cpio@2.13.90 prefix: /usr - cvs: + curl: externals: - - spec: cvs@1.11.23 + - spec: curl@7.76.1+gssapi+ldap+nghttp2 prefix: /usr diffutils: externals: - - spec: diffutils@3.6 + - spec: diffutils@3.7 prefix: /usr - doxygen: + dos2unix: externals: - - spec: doxygen@1.8.14+graphviz~mscgen + - spec: dos2unix@7.4.2 + prefix: /usr + file: + externals: + - spec: file@5.39 prefix: /usr findutils: externals: - - spec: findutils@4.6.0 + - spec: findutils@4.8.0 prefix: /usr flex: externals: - - spec: flex@2.6.1+lex + - spec: flex@2.6.4+lex prefix: /usr gawk: externals: - - spec: gawk@4.2.1 + - spec: gawk@5.1.0 prefix: /usr gettext: externals: - - spec: gettext@0.19.8.1 + - spec: gettext@0.21 + prefix: /usr + ghostscript: + externals: + - spec: ghostscript@9.54.0 prefix: /usr git: + buildable: False externals: - - spec: git@2.43.5+tcltk + - spec: git@2.47.1~tcltk prefix: /usr git-lfs: + buildable: False externals: - - spec: git-lfs@3.4.1 + - spec: git-lfs@3.6.1 prefix: /usr gmake: externals: - - spec: gmake@4.2.1 + - spec: gmake@4.3 + prefix: /usr + gnupg: + externals: + - spec: gnupg@2.3.3 prefix: /usr grep: externals: - - spec: grep@3.1 + - spec: grep@3.6 prefix: /usr groff: externals: - - spec: groff@1.22.3 + - spec: groff@1.22.4 + prefix: /usr + hwloc: + externals: + - spec: hwloc@2.4.1 + prefix: /usr + libfabric: + externals: + - spec: libfabric@2.1.0 fabrics=psm2,psm3,shm,tcp,verbs + prefix: /opt/intel/oneapi/mpi/2021.16 + - spec: libfabric@2.1.0 fabrics=psm2,psm3,shm,tcp,verbs + prefix: /opt/intel/oneapi/mpi/2021.16/opt/mpi/libfabric + - spec: libfabric@2.1.0 fabrics=psm2,psm3,shm,tcp,verbs prefix: /usr - libtool: + libfuse: externals: - - spec: libtool@2.4.6 + - spec: libfuse@2.9.9 + prefix: /usr + - spec: libfuse@3.10.2 + prefix: /usr + lustre: + externals: + - spec: lustre@2.15.6 prefix: /usr m4: externals: - - spec: m4@1.4.18 + - spec: m4@1.4.19 + prefix: /usr + mysql: + externals: + - spec: mysql@10.6.6 + prefix: /usr + ncurses: + externals: + - spec: ncurses@6.2.20210508+termlib abi=6 + prefix: /usr + nvidia-nsight-systems: + externals: + - spec: nvidia-nsight-systems@2025.1.3 + prefix: /usr/local + openblas: + externals: + - spec: openblas@0.3.26 + prefix: /usr + openjdk: + externals: + - spec: openjdk@17.0.15_6-LTS prefix: /usr openssh: externals: - - spec: openssh@8.0p1 + - spec: openssh@8.7p1 + prefix: /usr + openssl: + externals: + - spec: openssl@3.2.2 prefix: /usr perl: externals: - - spec: perl@5.26.3~cpanm+opcode+open+shared+threads + - spec: perl@5.32.1~cpanm+opcode+open+shared+threads prefix: /usr pkgconf: externals: - - spec: pkgconf@1.4.2 + - spec: pkgconf@1.7.3 prefix: /usr - qt: - buildable: False + rdma-core: + externals: + - spec: rdma-core@54.0 + prefix: /usr + rsync: + externals: + - spec: rsync@3.2.3 + prefix: /usr + ruby: externals: - - spec: qt@5.15.3 - prefix: /usr/lib64/qt5 + - spec: ruby@3.0.7 + prefix: /usr sed: externals: - - spec: sed@4.5 + - spec: sed@4.8 + prefix: /usr + slurm: + externals: + - spec: slurm@23.11.9 prefix: /usr - subversion: + sqlite: externals: - - spec: subversion@1.10.2 + - spec: sqlite@3.34.1+fts~functions+rtree prefix: /usr tar: externals: - - spec: tar@1.30 + - spec: tar@1.34 prefix: /usr texinfo: externals: - - spec: texinfo@6.5 + - spec: texinfo@6.7 prefix: /usr texlive: externals: - - spec: texlive@2018 + - spec: texlive@20200406 prefix: /usr wget: externals: - - spec: wget@1.19.5 + - spec: wget@1.21.1 + prefix: /usr + which: + externals: + - spec: which@2.21 + prefix: /usr + win-file: + externals: + - spec: win-file@5.39 + prefix: /usr + xz: + externals: + - spec: xz@5.2.5 + prefix: /usr + zip: + externals: + - spec: zip@3.0 prefix: /usr zlib: externals: diff --git a/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml b/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml new file mode 100644 index 000000000..28c59a9b1 --- /dev/null +++ b/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml @@ -0,0 +1,65 @@ +packages: + mpi: + buildable: False + require: + - "intel-oneapi-mpi@2021.16" + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2025.2.1 + prefix: /opt/intel/oneapi + modules: + - umf/0.11.0 + - tbb/2022.2 + - compiler-rt/2025.2.1 + - compiler/2025.2.1 + extra_attributes: + compilers: + c: /opt/intel/oneapi/compiler/2025.2/bin/icx + cxx: /opt/intel/oneapi/compiler/2025.2/bin/icpx + fortran: /opt/intel/oneapi/compiler/2025.2/bin/ifx + + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.16 + prefix: /opt/intel/oneapi + modules: + - mpi/2021.16 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.2 + prefix: /opt/intel/oneapi + modules: + - mkl/2025.2 + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2022.2 + prefix: /opt/intel/oneapi + modules: + - tbb/2022.2 + intel-oneapi-runtime: + ### ??? # LINK TO BUG REPORT HERE + buildable: false + externals: + - spec: intel-oneapi-runtime@2025.2.1 + prefix: /cm/shared/apps/intel/oneapi-2025.2.1 + modules: + - umf/0.11.0 + - tbb/2022.2 + - compiler-rt/2025.2.1 + - compiler/2025.2.1 + gcc: + externals: + - spec: gcc@11.5.0 languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@11.5.0 + prefix: /usr From 3be04dd7de6042892ef226aa6f28ef31316401fe Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 Aug 2025 14:44:34 -0600 Subject: [PATCH 058/163] In addition to py-beautifulsoup4, pin py-hatchling to newer version to work with newer python/python packages --- configs/common/packages.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 6cf7ab800..c82f2ec91 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -325,6 +325,11 @@ packages: py-h5py: require: - ~mpi + # Needed to fix issues of older versions (e.g. 1.13.0) with newer Python and + # newer python packages: "ValueError: Unknown classifier in field project.classifiers ..." + py-hatchling: + require: + - '@1.27:' # https://github.com/JCSDA/spack-stack/issues/1276 py-matplotlib: require: From b9948ab8d2ad9ec192ade6b26b567741ca66b54e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 Aug 2025 14:44:56 -0600 Subject: [PATCH 059/163] configs/common/packages_oneapi.yaml: build zlib-api with gcc --- configs/common/packages_oneapi.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index e9f98576f..557a550ec 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -48,4 +48,8 @@ packages: qt: require: - '%gcc' - + # Prevent issues with fault Intel libraries + # when using tar etc by building zlib-api with gcc + zlib-api: + require: + - '%gcc' From 34416a3f72da42763ae114e1ae4178c4e15e9c12 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 Aug 2025 14:46:22 -0600 Subject: [PATCH 060/163] Remove external zlib for atlantis, navy-aws, bounty --- configs/sites/tier1/atlantis/packages.yaml | 20 +++++++++----------- configs/sites/tier1/navy-aws/packages.yaml | 11 ++++------- configs/sites/tier2/bounty/packages.yaml | 11 ++++------- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/configs/sites/tier1/atlantis/packages.yaml b/configs/sites/tier1/atlantis/packages.yaml index bd98ec047..8fe277de0 100644 --- a/configs/sites/tier1/atlantis/packages.yaml +++ b/configs/sites/tier1/atlantis/packages.yaml @@ -1,12 +1,10 @@ packages: # Modification of common packages - # Prevent building zlib, zlib-ng - zlib-api: - buildable: false - # Pin py-torch to 2.5.1 to avoid build errors with 2.6.0 - py-torch: - require: - - '@2.5.1' + + ## Pin py-torch to 2.5.1 to avoid build errors with 2.6.0 + #py-torch: + # require: + # - '@2.5.1' # All other packages listed alphabetically autoconf: @@ -110,7 +108,7 @@ packages: externals: - spec: wget@1.19.5 prefix: /usr - zlib: - externals: - - spec: zlib@1.2.11 - prefix: /usr + #zlib: + # externals: + # - spec: zlib@1.2.11 + # prefix: /usr diff --git a/configs/sites/tier1/navy-aws/packages.yaml b/configs/sites/tier1/navy-aws/packages.yaml index 29e41fd75..e31c7b1ff 100644 --- a/configs/sites/tier1/navy-aws/packages.yaml +++ b/configs/sites/tier1/navy-aws/packages.yaml @@ -1,8 +1,5 @@ packages: # Modification of common packages - # Prevent building zlib, zlib-ng - zlib-api: - buildable: false # All other packages listed alphabetically autoconf: @@ -221,7 +218,7 @@ packages: externals: - spec: zip@3.0 prefix: /usr - zlib: - externals: - - spec: zlib@1.2.11 - prefix: /usr + #zlib: + # externals: + # - spec: zlib@1.2.11 + # prefix: /usr diff --git a/configs/sites/tier2/bounty/packages.yaml b/configs/sites/tier2/bounty/packages.yaml index a1b4ef1f9..b8206db38 100644 --- a/configs/sites/tier2/bounty/packages.yaml +++ b/configs/sites/tier2/bounty/packages.yaml @@ -1,8 +1,5 @@ packages: # Modification of common packages - # Prevent building zlib, zlib-ng - zlib-api: - buildable: false # All other packages listed alphabetically autoconf: @@ -106,7 +103,7 @@ packages: externals: - spec: wget@1.21.1 prefix: /usr - zlib: - externals: - - spec: zlib@1.2.11 - prefix: /usr + #zlib: + # externals: + # - spec: zlib@1.2.11 + # prefix: /usr From 8f526125d442400b9c46d77b40cd7224302aec72 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 Aug 2025 17:02:07 -0600 Subject: [PATCH 061/163] Update configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml and .github/workflows/ubuntu-ci-x86_64-gnu.yaml --- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 2 +- configs/sites/tier1/navy-aws/README.md | 2 + configs/sites/tier1/navy-aws/compilers.yaml | 28 ---- .../sites/tier1/navy-aws/packages_gcc.yaml | 15 -- .../navy-aws/packages_oneapi-2025.2.1.yaml | 130 +++++++++--------- 5 files changed, 68 insertions(+), 109 deletions(-) delete mode 100644 configs/sites/tier1/navy-aws/compilers.yaml delete mode 100644 configs/sites/tier1/navy-aws/packages_gcc.yaml diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 7840739a4..5fc66ce00 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -116,7 +116,7 @@ jobs: # Add binary cache if requested if [ "$USE_BINARY_CACHE" = true ] ; then set +e - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ + spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache-new-spack-v1/ spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) set -e echo "Packages in spack binary cache:" diff --git a/configs/sites/tier1/navy-aws/README.md b/configs/sites/tier1/navy-aws/README.md index b2b0e58f0..4adab23a4 100644 --- a/configs/sites/tier1/navy-aws/README.md +++ b/configs/sites/tier1/navy-aws/README.md @@ -1,5 +1,7 @@ # Provisiong NRL ParallelWorks AWS clusters +TODO UPDATE 20250820 !!! + ## Steps to perform before installing spack-stack develop as of 2024/11/24 Note. Some of these packages may already be installed, but for the sake of completeness, they are listed here. diff --git a/configs/sites/tier1/navy-aws/compilers.yaml b/configs/sites/tier1/navy-aws/compilers.yaml deleted file mode 100644 index 4c161e8cd..000000000 --- a/configs/sites/tier1/navy-aws/compilers.yaml +++ /dev/null @@ -1,28 +0,0 @@ -compilers: -- compiler: - spec: gcc@=13.3.1 - paths: - cc: /opt/rh/gcc-toolset-13/root/usr/bin/gcc - cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ - f77: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran - fc: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@=8.5.0 - paths: - cc: /usr/bin/gcc - cxx: /usr/bin/g++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - diff --git a/configs/sites/tier1/navy-aws/packages_gcc.yaml b/configs/sites/tier1/navy-aws/packages_gcc.yaml deleted file mode 100644 index b8f1b3c48..000000000 --- a/configs/sites/tier1/navy-aws/packages_gcc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -packages: - all: - compiler:: [gcc@13.3.1] - providers: - mpi:: [openmpi@5.0.5] - mpi: - buildable: False - openmpi: - buildable: False - externals: - - spec: openmpi@5.0.5%gcc@13.3.1 fabrics=auto ~gpfs ~internal-hwloc ~internal-libevent - ~internal-pmix +lustre +openshmem schedulers=slurm +romio romio-filesystem=lustre - modules: - - libfabric-aws/1.22.0amzn2.0 - - openmpi5/5.0.5 diff --git a/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml b/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml index 28c59a9b1..a2a784b4d 100644 --- a/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml +++ b/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml @@ -1,65 +1,65 @@ -packages: - mpi: - buildable: False - require: - - "intel-oneapi-mpi@2021.16" - intel-oneapi-compilers: - externals: - - spec: intel-oneapi-compilers@2025.2.1 - prefix: /opt/intel/oneapi - modules: - - umf/0.11.0 - - tbb/2022.2 - - compiler-rt/2025.2.1 - - compiler/2025.2.1 - extra_attributes: - compilers: - c: /opt/intel/oneapi/compiler/2025.2/bin/icx - cxx: /opt/intel/oneapi/compiler/2025.2/bin/icpx - fortran: /opt/intel/oneapi/compiler/2025.2/bin/ifx - - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.16 - prefix: /opt/intel/oneapi - modules: - - mpi/2021.16 - intel-oneapi-mkl: - buildable: false - externals: - - spec: intel-oneapi-mkl@2025.2 - prefix: /opt/intel/oneapi - modules: - - mkl/2025.2 - intel-oneapi-tbb: - buildable: false - externals: - - spec: intel-oneapi-tbb@2022.2 - prefix: /opt/intel/oneapi - modules: - - tbb/2022.2 - intel-oneapi-runtime: - ### ??? # LINK TO BUG REPORT HERE - buildable: false - externals: - - spec: intel-oneapi-runtime@2025.2.1 - prefix: /cm/shared/apps/intel/oneapi-2025.2.1 - modules: - - umf/0.11.0 - - tbb/2022.2 - - compiler-rt/2025.2.1 - - compiler/2025.2.1 - gcc: - externals: - - spec: gcc@11.5.0 languages:='c,c++' - prefix: /usr - extra_attributes: - compilers: - c: /usr/bin/gcc - cxx: /usr/bin/g++ - fortran: /usr/bin/gfortran - gcc-runtime: - externals: - - spec: gcc-runtime@11.5.0 - prefix: /usr +packages: + mpi: + buildable: False + require: + - "intel-oneapi-mpi@2021.16" + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2025.2.1 + prefix: /opt/intel/oneapi + modules: + - umf/0.11.0 + - tbb/2022.2 + - compiler-rt/2025.2.1 + - compiler/2025.2.1 + extra_attributes: + compilers: + c: /opt/intel/oneapi/compiler/2025.2/bin/icx + cxx: /opt/intel/oneapi/compiler/2025.2/bin/icpx + fortran: /opt/intel/oneapi/compiler/2025.2/bin/ifx + + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.16 + prefix: /opt/intel/oneapi + modules: + - mpi/2021.16 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.2 + prefix: /opt/intel/oneapi + modules: + - mkl/2025.2 + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2022.2 + prefix: /opt/intel/oneapi + modules: + - tbb/2022.2 + intel-oneapi-runtime: + # LINK TO BUG REPORT HERE + #buildable: false + externals: + - spec: intel-oneapi-runtime@2025.2.1 + prefix: /cm/shared/apps/intel/oneapi-2025.2.1 + modules: + - umf/0.11.0 + - tbb/2022.2 + - compiler-rt/2025.2.1 + - compiler/2025.2.1 + gcc: + externals: + - spec: gcc@11.5.0 languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@11.5.0 + prefix: /usr From ab0648a8a31d79e2b0df3101a6aa5a2939ebb3ca Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 Aug 2025 20:11:17 -0600 Subject: [PATCH 062/163] Update configs/sites/tier1/navy-aws/config.yaml and .github/workflows/ubuntu-ci-x86_64-gnu.yaml --- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 14 +++++++------- configs/sites/tier1/navy-aws/config.yaml | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 5fc66ce00..e92e17f3b 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -32,7 +32,7 @@ jobs: - name: prepare-directories run: | - mkdir -p /home/ubuntu/spack-stack/build-cache/ + mkdir -p /home/ubuntu/spack-stack/build-cache-new-spack-v1/ mkdir -p /home/ubuntu/spack-stack/source-cache/ - name: create-buildcache @@ -117,7 +117,7 @@ jobs: if [ "$USE_BINARY_CACHE" = true ] ; then set +e spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache-new-spack-v1/ - spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) + spack buildcache update-index local-binary set -e echo "Packages in spack binary cache:" spack buildcache list @@ -131,23 +131,23 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env + spack buildcache create -u local-binary base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u local-binary jedi-base-env # jedi-ufs-env echo "jedi-ufs-env ..." spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env + spack buildcache create -u local-binary jedi-ufs-env fi # the rest echo "${TEMPLATE} ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE} - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ + spack buildcache create -u local-binary # Remove binary cache for next round of concretization if [ "$USE_BINARY_CACHE" = true ] ; then @@ -181,7 +181,7 @@ jobs: ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env # Add binary cache back in - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ + spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache-new-spack-v1/ echo "Packages in combined spack build caches:" spack buildcache list diff --git a/configs/sites/tier1/navy-aws/config.yaml b/configs/sites/tier1/navy-aws/config.yaml index d01ba35fe..b30215cb2 100644 --- a/configs/sites/tier1/navy-aws/config.yaml +++ b/configs/sites/tier1/navy-aws/config.yaml @@ -4,7 +4,7 @@ config: # Overrides for spack build and staging areas to speed up builds # and avoid errors with hard links on the NFS filesystem /contrib - build_stage: /tmp/spack-stack/cache/build_stage - test_stage: /tmp/spack-stack/cache/test_stage - source_cache: /tmp/spack-stack/cache/source_cache - misc_cache: /tmp/spack-stack/cache/misc_cache + build_stage: /work/spack-stack-cache/build_stage + test_stage: /work/spack-stack-cache/test_stage + source_cache: /work/spack-stack-cache/source_cache + misc_cache: /work/spack-stack-cache/misc_cache From 31915816184df606e1a24cb4c12d2b04ff544205 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 21 Aug 2025 05:37:44 -0600 Subject: [PATCH 063/163] Build more envs in .github/workflows/ubuntu-ci-x86_64-gnu.yaml --- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index e92e17f3b..b2c760e44 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -51,8 +51,7 @@ jobs: # Set up spack-stack source ./setup.sh - #declare -a TEMPLATES=("unified-dev" "skylab-dev" "cylc-dev" "neptune-dev") - declare -a TEMPLATES=("cylc-dev" "neptune-dev") + declare -a TEMPLATES=("unified-dev" "skylab-dev" "cylc-dev" "neptune-dev") for TEMPLATE in "${TEMPLATES[@]}"; do if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then export ENVNAME=ue-gcc-11.4.0-buildcache @@ -106,7 +105,7 @@ jobs: fi # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.gnu-11.4.0-buildcache + spack concretize 2>&1 | tee log.concretize.${ENVNAME} ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env # Add and update source cache @@ -130,23 +129,23 @@ jobs: if [[ "${TEMPLATE}" == *"unified-dev"* || "${TEMPLATE}" == *"skylab-dev"* ]]; then # base-env echo "base-env ..." - spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env + spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env spack buildcache create -u local-binary base-env # jedi-base-env echo "jedi-base-env ..." - spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env + spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.${ENVNAME}.jedi-base-env spack buildcache create -u local-binary jedi-base-env # jedi-ufs-env echo "jedi-ufs-env ..." - spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.${ENVNAME}.jedi-ufs-env spack buildcache create -u local-binary jedi-ufs-env fi # the rest echo "${TEMPLATE} ..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE} + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${ENVNAME}.all spack buildcache create -u local-binary # Remove binary cache for next round of concretization @@ -177,7 +176,7 @@ jobs: spack env activate ${ENVDIR} # Concretize and check for duplicates - spack concretize --force 2>&1 | tee log.concretize.gnu-11.4.0 + spack concretize --force 2>&1 | tee log.concretize.${ENVNAME} ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env # Add binary cache back in @@ -186,7 +185,7 @@ jobs: spack buildcache list # Install from cache - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0.unified-env + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${ENVNAME}.all # Check shared libraries ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check @@ -226,7 +225,7 @@ jobs: module use ${ENVDIR}/install/modulefiles/Core module load stack-gcc/11.4.0 - module load stack-openmpi/5.0.6 + module load stack-openmpi/5.0.8 module load stack-python/3.11.11 module available From 460f26605884241a4700f7af16b1eb02e70426a1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 21 Aug 2025 06:58:32 -0600 Subject: [PATCH 064/163] Move outdated templates to temporary subdirectory; add new template neptune-ops; make esmf python variant configurable at the template level --- configs/common/packages.yaml | 2 +- .../ccpp-scm/spack.yaml | 0 .../gfs-v16.2/spack.yaml | 0 .../gsi-addon-dev/spack.yaml | 0 .../jedi-mpas-nvidia-dev/spack.yaml | 0 .../nco/approved_list.txt | 0 .../nco/spack.yaml | 0 .../ufs-srw-dev/spack.yaml | 0 .../ufs-srw-public-v2/spack.yaml | 0 .../ufs-utils/spack.yaml | 0 .../ufs-weather-model-static/spack.yaml | 0 .../ufs-weather-model/spack.yaml | 0 configs/templates/empty/spack.yaml | 11 +-- configs/templates/neptune-dev/spack.yaml | 7 +- configs/templates/neptune-ops/spack.yaml | 21 +++++ configs/templates/skylab-dev/spack.yaml | 80 ++++++++----------- configs/templates/unified-dev/spack.yaml | 11 ++- 17 files changed, 71 insertions(+), 61 deletions(-) rename configs/templates/{ => NOT_YET_CONVERTED_TO_SPACK_v1}/ccpp-scm/spack.yaml (100%) rename configs/templates/{ => NOT_YET_CONVERTED_TO_SPACK_v1}/gfs-v16.2/spack.yaml (100%) rename configs/templates/{ => NOT_YET_CONVERTED_TO_SPACK_v1}/gsi-addon-dev/spack.yaml (100%) rename configs/templates/{ => NOT_YET_CONVERTED_TO_SPACK_v1}/jedi-mpas-nvidia-dev/spack.yaml (100%) rename configs/templates/{ => NOT_YET_CONVERTED_TO_SPACK_v1}/nco/approved_list.txt (100%) rename configs/templates/{ => NOT_YET_CONVERTED_TO_SPACK_v1}/nco/spack.yaml (100%) rename configs/templates/{ => NOT_YET_CONVERTED_TO_SPACK_v1}/ufs-srw-dev/spack.yaml (100%) rename configs/templates/{ => NOT_YET_CONVERTED_TO_SPACK_v1}/ufs-srw-public-v2/spack.yaml (100%) rename configs/templates/{ => NOT_YET_CONVERTED_TO_SPACK_v1}/ufs-utils/spack.yaml (100%) rename configs/templates/{ => NOT_YET_CONVERTED_TO_SPACK_v1}/ufs-weather-model-static/spack.yaml (100%) rename configs/templates/{ => NOT_YET_CONVERTED_TO_SPACK_v1}/ufs-weather-model/spack.yaml (100%) create mode 100644 configs/templates/neptune-ops/spack.yaml diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index c82f2ec91..132cf006a 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -83,7 +83,7 @@ packages: # Also, check the acorn and derecho site configs which have esmf modifications. esmf: require: - - ~xerces ~pnetcdf +shared +external-parallelio +python + - ~xerces ~pnetcdf +shared +external-parallelio - any_of: ['@=8.6.1 snapshot=none', '@=8.8.0 snapshot=none', '@=8.9.0 snapshot=none'] - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] when: '%intel' diff --git a/configs/templates/ccpp-scm/spack.yaml b/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ccpp-scm/spack.yaml similarity index 100% rename from configs/templates/ccpp-scm/spack.yaml rename to configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ccpp-scm/spack.yaml diff --git a/configs/templates/gfs-v16.2/spack.yaml b/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/gfs-v16.2/spack.yaml similarity index 100% rename from configs/templates/gfs-v16.2/spack.yaml rename to configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/gfs-v16.2/spack.yaml diff --git a/configs/templates/gsi-addon-dev/spack.yaml b/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/gsi-addon-dev/spack.yaml similarity index 100% rename from configs/templates/gsi-addon-dev/spack.yaml rename to configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/gsi-addon-dev/spack.yaml diff --git a/configs/templates/jedi-mpas-nvidia-dev/spack.yaml b/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/jedi-mpas-nvidia-dev/spack.yaml similarity index 100% rename from configs/templates/jedi-mpas-nvidia-dev/spack.yaml rename to configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/jedi-mpas-nvidia-dev/spack.yaml diff --git a/configs/templates/nco/approved_list.txt b/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/nco/approved_list.txt similarity index 100% rename from configs/templates/nco/approved_list.txt rename to configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/nco/approved_list.txt diff --git a/configs/templates/nco/spack.yaml b/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/nco/spack.yaml similarity index 100% rename from configs/templates/nco/spack.yaml rename to configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/nco/spack.yaml diff --git a/configs/templates/ufs-srw-dev/spack.yaml b/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-srw-dev/spack.yaml similarity index 100% rename from configs/templates/ufs-srw-dev/spack.yaml rename to configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-srw-dev/spack.yaml diff --git a/configs/templates/ufs-srw-public-v2/spack.yaml b/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-srw-public-v2/spack.yaml similarity index 100% rename from configs/templates/ufs-srw-public-v2/spack.yaml rename to configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-srw-public-v2/spack.yaml diff --git a/configs/templates/ufs-utils/spack.yaml b/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-utils/spack.yaml similarity index 100% rename from configs/templates/ufs-utils/spack.yaml rename to configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-utils/spack.yaml diff --git a/configs/templates/ufs-weather-model-static/spack.yaml b/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-weather-model-static/spack.yaml similarity index 100% rename from configs/templates/ufs-weather-model-static/spack.yaml rename to configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-weather-model-static/spack.yaml diff --git a/configs/templates/ufs-weather-model/spack.yaml b/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-weather-model/spack.yaml similarity index 100% rename from configs/templates/ufs-weather-model/spack.yaml rename to configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-weather-model/spack.yaml diff --git a/configs/templates/empty/spack.yaml b/configs/templates/empty/spack.yaml index e95a6dd89..e8478df51 100644 --- a/configs/templates/empty/spack.yaml +++ b/configs/templates/empty/spack.yaml @@ -1,16 +1,9 @@ +# Empty template for testing, chained environments, and developing new templates +# spack: concretizer: unify: when_possible - config: - install_tree: - root: $env/install - modules: - default: - roots: - lmod: $env/install/modulefiles - tcl: $env/install/modulefiles - view: false include: [] diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 8f58388d3..b760d51b9 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -1,3 +1,5 @@ +# Template for NEPTUNE development +# spack: concretizer: unify: when_possible @@ -12,9 +14,12 @@ spack: - jedi-neptune-env ^neptune-env ~debug +espc ^esmf@=8.9.0 - crtm@3.1.2 - crtm@v2.4.1-jedi.2 - - esmf@8.9.0 packages: + # Turn on python variant for esmf + esmf: + require: + - +python # Avoid duplicate builds of parallelio +pnetcdf and ~pnetcdf parallelio: require: diff --git a/configs/templates/neptune-ops/spack.yaml b/configs/templates/neptune-ops/spack.yaml new file mode 100644 index 000000000..4313ec690 --- /dev/null +++ b/configs/templates/neptune-ops/spack.yaml @@ -0,0 +1,21 @@ +# Template for NEPTUNE operations +# +spack: + concretizer: + unify: when_possible + + view: false + include: [] + + specs: + - neptune-env ~debug +espc ^esmf@=8.9.0 + + packages: + # Turn off python variant for esmf + esmf: + require: + - ~python + # Avoid duplicate builds of parallelio +pnetcdf and ~pnetcdf + parallelio: + require: + - ~pnetcdf diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 560c2900a..ed037311f 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -1,3 +1,5 @@ +# Template for JEDI Skylab development +# spack: concretizer: unify: when_possible @@ -5,56 +7,40 @@ spack: view: false include: [] - definitions: - - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - - packages: - - ewok-env +ecflow ~cylc - - ai-env - - geos-gcm-env ^esmf@=8.6.1 - - jedi-fv3-env - - jedi-geos-env ^esmf@=8.6.1 - - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.9.0 - - jedi-tools-env - - jedi-ufs-env ^esmf@=8.8.0 - - jedi-um-env - - neptune-env ^esmf@=8.9.0 - - neptune-python-env ^esmf@=8.9.0 - - soca-env - - # Various crtm tags (list all to avoid duplicate packages) - - crtm@2.4.0.1 - - crtm@v2.4.1-jedi.2 - - crtm@3.1.2 - - # Mapl with various esmf tags - - mapl@2.53.0 ^esmf@8.6.1 - - mapl@2.53.4 ^esmf@8.8.0 - - # Various esmf tags (list all to avoid duplicate packages) - - esmf@=8.6.1 - - esmf@=8.8.0 - - esmf@=8.9.0 - specs: - - matrix: - - [$packages] - - [$compilers] - exclude: - - # Exclude Intel and OneAPI variants - - jedi-tools-env%intel - - jedi-tools-env%oneapi - - # py-torch in ai-env doesn't build with Intel - - ai-env%intel - - ai-env%oneapi - - # Skip incompatible CRTM versions with oneapi - - crtm@2.4.0.1%oneapi + - ewok-env +ecflow ~cylc + #- ai-env + - geos-gcm-env ^esmf@=8.6.1 + - jedi-fv3-env + - jedi-geos-env ^esmf@=8.6.1 + - jedi-mpas-env + - jedi-neptune-env ^esmf@=8.9.0 + - jedi-tools-env + - jedi-ufs-env ^esmf@=8.8.0 + #- jedi-um-env + - neptune-env ^esmf@=8.9.0 + - neptune-python-env ^esmf@=8.9.0 + - soca-env + + # Various crtm tags (list all to avoid duplicate packages) + - crtm@v2.4.1-jedi.2 + - crtm@3.1.2 + + # Mapl with various esmf tags + - mapl@2.53.0 ^esmf@8.6.1 + - mapl@2.53.4 ^esmf@8.8.0 + + # Various esmf tags (list all to avoid duplicate packages) + - esmf@=8.6.1 + - esmf@=8.8.0 + - esmf@=8.9.0 packages: + # Turn on python variant for esmf + esmf: + require: + - +python # Avoid duplicate builds of parallelio +pnetcdf and ~pnetcdf parallelio: require: - - '+pnetcdf' + - +pnetcdf diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index fc0db0b61..5e1b02258 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -1,3 +1,5 @@ +# Template for grand, unified environment +# spack: concretizer: unify: when_possible @@ -18,7 +20,7 @@ spack: - jedi-neptune-env ^esmf@=8.9.0 #- jedi-tools-env - jedi-ufs-env ^esmf@=8.8.0 - - jedi-um-env + #- jedi-um-env - neptune-env ^esmf@=8.9.0 - neptune-python-env ^esmf@=8.9.0 - soca-env @@ -26,7 +28,6 @@ spack: - ufs-weather-model-env ^esmf@=8.8.0 ^crtm@=3.1.2 # Various crtm tags (list all to avoid duplicate packages) - #- crtm@2.4.0.1 - crtm@v2.4.1-jedi.2 - crtm@3.1.2 @@ -43,7 +44,11 @@ spack: - madis@4.5 packages: + # Turn on python variant for esmf + esmf: + require: + - +python # Avoid duplicate builds of parallelio +pnetcdf and ~pnetcdf parallelio: require: - - '+pnetcdf' + - +pnetcdf From b519d7a0f84b32bea1d50bf646968e53cc06e4c2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 21 Aug 2025 12:42:30 -0600 Subject: [PATCH 065/163] configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1 --> configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1 --- .../ccpp-scm/spack.yaml | 0 .../gfs-v16.2/spack.yaml | 0 .../gsi-addon-dev/spack.yaml | 0 .../jedi-mpas-nvidia-dev/spack.yaml | 0 .../nco/approved_list.txt | 0 .../nco/spack.yaml | 0 .../ufs-srw-dev/spack.yaml | 0 .../ufs-srw-public-v2/spack.yaml | 0 .../ufs-utils/spack.yaml | 0 .../ufs-weather-model-static/spack.yaml | 0 .../ufs-weather-model/spack.yaml | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename configs/{templates/NOT_YET_CONVERTED_TO_SPACK_v1 => templates_NOT_YET_CONVERTED_TO_SPACK_v1}/ccpp-scm/spack.yaml (100%) rename configs/{templates/NOT_YET_CONVERTED_TO_SPACK_v1 => templates_NOT_YET_CONVERTED_TO_SPACK_v1}/gfs-v16.2/spack.yaml (100%) rename configs/{templates/NOT_YET_CONVERTED_TO_SPACK_v1 => templates_NOT_YET_CONVERTED_TO_SPACK_v1}/gsi-addon-dev/spack.yaml (100%) rename configs/{templates/NOT_YET_CONVERTED_TO_SPACK_v1 => templates_NOT_YET_CONVERTED_TO_SPACK_v1}/jedi-mpas-nvidia-dev/spack.yaml (100%) rename configs/{templates/NOT_YET_CONVERTED_TO_SPACK_v1 => templates_NOT_YET_CONVERTED_TO_SPACK_v1}/nco/approved_list.txt (100%) rename configs/{templates/NOT_YET_CONVERTED_TO_SPACK_v1 => templates_NOT_YET_CONVERTED_TO_SPACK_v1}/nco/spack.yaml (100%) rename configs/{templates/NOT_YET_CONVERTED_TO_SPACK_v1 => templates_NOT_YET_CONVERTED_TO_SPACK_v1}/ufs-srw-dev/spack.yaml (100%) rename configs/{templates/NOT_YET_CONVERTED_TO_SPACK_v1 => templates_NOT_YET_CONVERTED_TO_SPACK_v1}/ufs-srw-public-v2/spack.yaml (100%) rename configs/{templates/NOT_YET_CONVERTED_TO_SPACK_v1 => templates_NOT_YET_CONVERTED_TO_SPACK_v1}/ufs-utils/spack.yaml (100%) rename configs/{templates/NOT_YET_CONVERTED_TO_SPACK_v1 => templates_NOT_YET_CONVERTED_TO_SPACK_v1}/ufs-weather-model-static/spack.yaml (100%) rename configs/{templates/NOT_YET_CONVERTED_TO_SPACK_v1 => templates_NOT_YET_CONVERTED_TO_SPACK_v1}/ufs-weather-model/spack.yaml (100%) diff --git a/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ccpp-scm/spack.yaml b/configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/ccpp-scm/spack.yaml similarity index 100% rename from configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ccpp-scm/spack.yaml rename to configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/ccpp-scm/spack.yaml diff --git a/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/gfs-v16.2/spack.yaml b/configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/gfs-v16.2/spack.yaml similarity index 100% rename from configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/gfs-v16.2/spack.yaml rename to configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/gfs-v16.2/spack.yaml diff --git a/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/gsi-addon-dev/spack.yaml b/configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/gsi-addon-dev/spack.yaml similarity index 100% rename from configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/gsi-addon-dev/spack.yaml rename to configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/gsi-addon-dev/spack.yaml diff --git a/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/jedi-mpas-nvidia-dev/spack.yaml b/configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/jedi-mpas-nvidia-dev/spack.yaml similarity index 100% rename from configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/jedi-mpas-nvidia-dev/spack.yaml rename to configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/jedi-mpas-nvidia-dev/spack.yaml diff --git a/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/nco/approved_list.txt b/configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/nco/approved_list.txt similarity index 100% rename from configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/nco/approved_list.txt rename to configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/nco/approved_list.txt diff --git a/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/nco/spack.yaml b/configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/nco/spack.yaml similarity index 100% rename from configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/nco/spack.yaml rename to configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/nco/spack.yaml diff --git a/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-srw-dev/spack.yaml b/configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/ufs-srw-dev/spack.yaml similarity index 100% rename from configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-srw-dev/spack.yaml rename to configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/ufs-srw-dev/spack.yaml diff --git a/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-srw-public-v2/spack.yaml b/configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/ufs-srw-public-v2/spack.yaml similarity index 100% rename from configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-srw-public-v2/spack.yaml rename to configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/ufs-srw-public-v2/spack.yaml diff --git a/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-utils/spack.yaml b/configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/ufs-utils/spack.yaml similarity index 100% rename from configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-utils/spack.yaml rename to configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/ufs-utils/spack.yaml diff --git a/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-weather-model-static/spack.yaml b/configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/ufs-weather-model-static/spack.yaml similarity index 100% rename from configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-weather-model-static/spack.yaml rename to configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/ufs-weather-model-static/spack.yaml diff --git a/configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-weather-model/spack.yaml b/configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/ufs-weather-model/spack.yaml similarity index 100% rename from configs/templates/NOT_YET_CONVERTED_TO_SPACK_v1/ufs-weather-model/spack.yaml rename to configs/templates_NOT_YET_CONVERTED_TO_SPACK_v1/ufs-weather-model/spack.yaml From 78e5593b18d38b2be65682c44e0cbdb123d9c721 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 21 Aug 2025 12:55:01 -0600 Subject: [PATCH 066/163] Update tier2 site config for Blackpearl --- .../sites/tier2/blackpearl/compilers.yaml.old | 72 ------------------- configs/sites/tier2/blackpearl/packages.yaml | 19 +++-- .../tier2/blackpearl/packages_gcc-13.2.1.yaml | 2 +- 3 files changed, 10 insertions(+), 83 deletions(-) delete mode 100644 configs/sites/tier2/blackpearl/compilers.yaml.old diff --git a/configs/sites/tier2/blackpearl/compilers.yaml.old b/configs/sites/tier2/blackpearl/compilers.yaml.old deleted file mode 100644 index 13e655e21..000000000 --- a/configs/sites/tier2/blackpearl/compilers.yaml.old +++ /dev/null @@ -1,72 +0,0 @@ -compilers: -- compiler: - spec: gcc@=11.4.1 - paths: - cc: /usr/bin/gcc - cxx: /usr/bin/g++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - flags: {} - operating_system: oracle9 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@=13.3.0 - paths: - cc: /home/dom/prod/gcc-13.3.0-manual/bin/gcc - cxx: /home/dom/prod/gcc-13.3.0-manual/bin/g++ - f77: /home/dom/prod/gcc-13.3.0-manual/bin/gfortran - fc: /home/dom/prod/gcc-13.3.0-manual/bin/gfortran - flags: {} - operating_system: oracle9 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: oneapi@=2024.2.1 - paths: - cc: /opt/intel/oneapi/2024.2/bin/icx - cxx: /opt/intel/oneapi/2024.2/bin/icpx - f77: /opt/intel/oneapi/2024.2/bin/ifort - fc: /opt/intel/oneapi/2024.2/bin/ifort - flags: {} - operating_system: oracle9 - target: x86_64 - modules: [] - environment: - append_path: - CPATH: /opt/intel/oneapi/compiler/2024.2/opt/compiler/include/intel64 - extra_rpaths: [] -- compiler: - spec: oneapi@=2025.1.0 - paths: - cc: /opt/intel/oneapi/2025.1/bin/icx - cxx: /opt/intel/oneapi/2025.1/bin/icpx - f77: /opt/intel/oneapi/2025.1/bin/ifx - fc: /opt/intel/oneapi/2025.1/bin/ifx - flags: {} - operating_system: oracle9 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: aocc@=4.2.0 - paths: - cc: /home/dom/prod/aocc-4.2.0/bin/clang - cxx: /home/dom/prod/aocc-4.2.0/bin/clang++ - f77: /home/dom/prod/aocc-4.2.0/bin/flang - fc: /home/dom/prod/aocc-4.2.0/bin/flang - flags: {} - operating_system: oracle9 - target: x86_64 - modules: - - aocc/4.2.0 - environment: - append_path: - # For libquadmath etc - LD_LIBRARY_PATH: /usr/lib/gcc/x86_64-redhat-linux/11 - extra_rpaths: [] diff --git a/configs/sites/tier2/blackpearl/packages.yaml b/configs/sites/tier2/blackpearl/packages.yaml index 495587560..26e83dece 100644 --- a/configs/sites/tier2/blackpearl/packages.yaml +++ b/configs/sites/tier2/blackpearl/packages.yaml @@ -1,12 +1,9 @@ packages: # Modification of common packages - # Prevent building zlib, zlib-ng - zlib-api: - buildable: false # Pin py-torch to 2.5.1 to avoid build errors with 2.6.0 - py-torch: - require: - - '@2.5.1' + #py-torch: + # require: + # - '@2.5.1' # All other packages listed alphabetically autoconf: @@ -50,10 +47,12 @@ packages: - spec: gettext@0.21 prefix: /usr git: + buildable: False externals: - spec: git@2.43.0~tcltk prefix: /usr git-lfs: + buildable: False externals: - spec: git-lfs@3.4.1 prefix: /usr @@ -90,7 +89,7 @@ packages: externals: - spec: wget@1.21.1 prefix: /usr - zlib: - externals: - - spec: zlib@1.2.11 - prefix: /usr + #zlib: + # externals: + # - spec: zlib@1.2.11 + # prefix: /usr diff --git a/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml b/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml index f5b2423cd..05ea6c755 100644 --- a/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml +++ b/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml @@ -1,7 +1,7 @@ packages: mpi: require: - - "openmpi@=5.0.6" + - "openmpi@=5.0.8" gcc: externals: - spec: gcc@13.2.1 languages:='c,c++,fortran' From 7dd6bb710d0202f2acb033846079428f01035bec Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 22 Aug 2025 08:58:46 -0600 Subject: [PATCH 067/163] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index bb6937004..7b51e1463 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit bb6937004a7bdc9f059b72c40587259dae2afee3 +Subproject commit 7b51e1463a860bc8da8cd352515e0f7d993c8ddf From 00c3f9e5daf9f9902bce701d9b6530e3af053a2c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 22 Aug 2025 09:29:34 -0600 Subject: [PATCH 068/163] configs/common/packages_oneapi.yaml: force building zlib with gcc, and set zlib-ng to buildable: false --- configs/common/packages_oneapi.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index e5209b95b..c890fd2fd 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -53,3 +53,5 @@ packages: zlib: require: - '%gcc' + zlib-ng: + buildable: False From 896423df5e715059c84a0500ae36d91ed68dc117 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 22 Aug 2025 09:29:53 -0600 Subject: [PATCH 069/163] configs/sites/tier2/bounty/packages_clang-20.1.5.yaml: add missing extra_rpaths --- configs/sites/tier2/bounty/packages_clang-20.1.5.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml b/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml index 25d013f79..df08bf982 100644 --- a/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml +++ b/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml @@ -11,3 +11,5 @@ packages: c: /home/dom/prod/llvm-20.1.5/bin/clang cxx: /home/dom/prod/llvm-20.1.5/bin/clang++ fortran: /home/dom/prod/llvm-20.1.5/bin/flang-new + extra_rpaths: + - /home/dom/prod/llvm-20.1.5/lib From 67dc0b18427d18c63a342a772a65f35e20e4dd38 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 22 Aug 2025 09:30:06 -0600 Subject: [PATCH 070/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 04a23bb01..d2e8ad549 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 04a23bb01a02e775a2a9b7863fe34487f9be25b0 +Subproject commit d2e8ad549e7603b22cad82366599d734bfb1554b From 236dba4388ee38bb183a4cf2e0ab42db35bc207a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 25 Aug 2025 11:39:42 +0000 Subject: [PATCH 071/163] Add configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml and configs/sites/tier1/navy-aws/packages_oneapi-2025.1.1.yaml --- .../navy-aws/packages_oneapi-2024.2.1.yaml | 63 ++++++++++++++++++ .../navy-aws/packages_oneapi-2025.1.1.yaml | 65 +++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml create mode 100644 configs/sites/tier1/navy-aws/packages_oneapi-2025.1.1.yaml diff --git a/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml new file mode 100644 index 000000000..263b411d5 --- /dev/null +++ b/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml @@ -0,0 +1,63 @@ +packages: + mpi: + buildable: False + require: + - "intel-oneapi-mpi@2021.13" + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2024.2.1 + prefix: /contrib/oneapi/2024.2.1 + modules: + - tbb/202.13 + - compiler-rt/2024.2.1 + - compiler/2024.2.1 + extra_attributes: + compilers: + c: /contrib/oneapi/2024.2.1/compiler/2024.2/bin/icx + cxx: /contrib/oneapi/2024.2.1/compiler/2024.2/bin/icpx + fortran: /contrib/oneapi/2024.2.1/compiler/2024.2/bin/ifx + + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.13 + prefix: /contrib/oneapi/2024.2.1 + modules: + - mpi/2021.13 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2 + prefix: /contrib/oneapi/2024.2.1 + modules: + - mkl/2024.2 + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2021.13 + prefix: /contrib/oneapi/2024.2.1 + modules: + - tbb/2021.13 + intel-oneapi-runtime: + # LINK TO BUG REPORT HERE + #buildable: false + externals: + - spec: intel-oneapi-runtime@2024.2.1 + prefix: /contrib/oneapi/2024.2.1 + modules: + - tbb/2021.13 + - compiler-rt/2024.2.1 + - compiler/2024.2.1 + gcc: + externals: + - spec: gcc@11.5.0 languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@11.5.0 + prefix: /usr diff --git a/configs/sites/tier1/navy-aws/packages_oneapi-2025.1.1.yaml b/configs/sites/tier1/navy-aws/packages_oneapi-2025.1.1.yaml new file mode 100644 index 000000000..af81fd950 --- /dev/null +++ b/configs/sites/tier1/navy-aws/packages_oneapi-2025.1.1.yaml @@ -0,0 +1,65 @@ +packages: + mpi: + buildable: False + require: + - "intel-oneapi-mpi@2021.15" + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2025.1.1 + prefix: /contrib/oneapi/2025.1.1 + modules: + - umf/0.10.0 + - tbb/2022.1 + - compiler-rt/2025.1.1 + - compiler/2025.1.1 + extra_attributes: + compilers: + c: /contrib/oneapi/2025.1.1/compiler/2025.1/bin/icx + cxx: /contrib/oneapi/2025.1.1/compiler/2025.1/bin/icpx + fortran: /contrib/oneapi/2025.1.1/compiler/2025.1/bin/ifx + + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.15 + prefix: /contrib/oneapi/2025.1.1 + modules: + - mpi/2021.15 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.1 + prefix: /contrib/oneapi/2025.1.1 + modules: + - mkl/2025.1 + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2022.2 + prefix: /contrib/oneapi/2025.1.1 + modules: + - tbb/2022.1 + intel-oneapi-runtime: + # LINK TO BUG REPORT HERE + #buildable: false + externals: + - spec: intel-oneapi-runtime@2025.1.1 + prefix: /contrib/oneapi/2025.1.1 + modules: + - umf/0.10.0 + - tbb/2022.1 + - compiler-rt/2025.1.1 + - compiler/2025.1.1 + gcc: + externals: + - spec: gcc@11.5.0 languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@11.5.0 + prefix: /usr From e7219fa1677c388dbbde82cac22227699591a377 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 25 Aug 2025 11:40:33 +0000 Subject: [PATCH 072/163] [skip ci] From 8e06a4af145b2e942e4672d3ad3dc29715902904 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 25 Aug 2025 13:25:09 -0600 Subject: [PATCH 073/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index d2e8ad549..f4c704bb7 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit d2e8ad549e7603b22cad82366599d734bfb1554b +Subproject commit f4c704bb74f239687ef6199ee8730e23f953d90f From 443ddae46e95b3fb2d40ecbfbe3b2e265b98995c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 25 Aug 2025 19:26:30 +0000 Subject: [PATCH 074/163] [skip ci] Remove configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml and update ifx->ifort in configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml --- .../navy-aws/packages_oneapi-2024.2.1.yaml | 3 +- .../navy-aws/packages_oneapi-2025.2.1.yaml | 65 ------------------- 2 files changed, 1 insertion(+), 67 deletions(-) delete mode 100644 configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml diff --git a/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml index 263b411d5..c974e17c0 100644 --- a/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml +++ b/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml @@ -15,8 +15,7 @@ packages: compilers: c: /contrib/oneapi/2024.2.1/compiler/2024.2/bin/icx cxx: /contrib/oneapi/2024.2.1/compiler/2024.2/bin/icpx - fortran: /contrib/oneapi/2024.2.1/compiler/2024.2/bin/ifx - + fortran: /contrib/oneapi/2024.2.1/compiler/2024.2/bin/ifort intel-oneapi-mpi: buildable: false externals: diff --git a/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml b/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml deleted file mode 100644 index a2a784b4d..000000000 --- a/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.1.yaml +++ /dev/null @@ -1,65 +0,0 @@ -packages: - mpi: - buildable: False - require: - - "intel-oneapi-mpi@2021.16" - intel-oneapi-compilers: - externals: - - spec: intel-oneapi-compilers@2025.2.1 - prefix: /opt/intel/oneapi - modules: - - umf/0.11.0 - - tbb/2022.2 - - compiler-rt/2025.2.1 - - compiler/2025.2.1 - extra_attributes: - compilers: - c: /opt/intel/oneapi/compiler/2025.2/bin/icx - cxx: /opt/intel/oneapi/compiler/2025.2/bin/icpx - fortran: /opt/intel/oneapi/compiler/2025.2/bin/ifx - - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.16 - prefix: /opt/intel/oneapi - modules: - - mpi/2021.16 - intel-oneapi-mkl: - buildable: false - externals: - - spec: intel-oneapi-mkl@2025.2 - prefix: /opt/intel/oneapi - modules: - - mkl/2025.2 - intel-oneapi-tbb: - buildable: false - externals: - - spec: intel-oneapi-tbb@2022.2 - prefix: /opt/intel/oneapi - modules: - - tbb/2022.2 - intel-oneapi-runtime: - # LINK TO BUG REPORT HERE - #buildable: false - externals: - - spec: intel-oneapi-runtime@2025.2.1 - prefix: /cm/shared/apps/intel/oneapi-2025.2.1 - modules: - - umf/0.11.0 - - tbb/2022.2 - - compiler-rt/2025.2.1 - - compiler/2025.2.1 - gcc: - externals: - - spec: gcc@11.5.0 languages:='c,c++' - prefix: /usr - extra_attributes: - compilers: - c: /usr/bin/gcc - cxx: /usr/bin/g++ - fortran: /usr/bin/gfortran - gcc-runtime: - externals: - - spec: gcc-runtime@11.5.0 - prefix: /usr From 046a2b03d0587767aa34e2f83dab9dd9ddbd20c1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 25 Aug 2025 13:37:25 -0600 Subject: [PATCH 075/163] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 7b51e1463..dc8e38807 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 7b51e1463a860bc8da8cd352515e0f7d993c8ddf +Subproject commit dc8e38807900b193d8f826533db5ded1340d4f05 From c9602327944721892193eded16caaada8d978f07 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 26 Aug 2025 05:30:41 -0600 Subject: [PATCH 076/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index f4c704bb7..ba4d68ea7 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit f4c704bb74f239687ef6199ee8730e23f953d90f +Subproject commit ba4d68ea7f87295fee485917e1ac679b425506b8 From 8a5c50463a1f9eb9c611d41217d3d5795f14e058 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 26 Aug 2025 07:29:22 -0600 Subject: [PATCH 077/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index ba4d68ea7..897305673 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit ba4d68ea7f87295fee485917e1ac679b425506b8 +Subproject commit 89730567304a3268713b93bd21726ae312303796 From ee3463732290189661b6fed0f7b4d6ad21559a9c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 26 Aug 2025 08:52:31 -0600 Subject: [PATCH 078/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 897305673..87e4b04a0 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 89730567304a3268713b93bd21726ae312303796 +Subproject commit 87e4b04a04079e0c496179f8d0b6157fe707d825 From 36c0c94a8a0240a34873bf80cd8f6faa6c04faf1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 26 Aug 2025 19:04:20 +0000 Subject: [PATCH 079/163] Fix wrong tbb module name in configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml --- configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml index c974e17c0..ea9f04880 100644 --- a/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml +++ b/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml @@ -8,7 +8,7 @@ packages: - spec: intel-oneapi-compilers@2024.2.1 prefix: /contrib/oneapi/2024.2.1 modules: - - tbb/202.13 + - tbb/2021.13 - compiler-rt/2024.2.1 - compiler/2024.2.1 extra_attributes: From 5ebe8a23160700e02669ce77950104602f79fb85 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 26 Aug 2025 19:05:14 +0000 Subject: [PATCH 080/163] [skip ci] From 7e890e9fbbf32d63c735f62c47c66b4fe6ee9839 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 26 Aug 2025 19:09:44 -0600 Subject: [PATCH 081/163] Update configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml and configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml --- .../tier1/atlantis/packages_clang-21.1.0.yaml | 23 +++++++++++++++++++ .../packages_clang.yaml.notyetupdated | 15 ------------ .../tier1/atlantis/packages_gcc-13.4.0.yaml | 14 +++++------ 3 files changed, 30 insertions(+), 22 deletions(-) create mode 100644 configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml delete mode 100644 configs/sites/tier1/atlantis/packages_clang.yaml.notyetupdated diff --git a/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml b/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml new file mode 100644 index 000000000..4b769f5a7 --- /dev/null +++ b/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml @@ -0,0 +1,23 @@ +packages: + mpi: + buildable: False + require: + - "openmpi@4.1.8" + llvm: + externals: + - spec: llvm@21.1.0 +flang ~libomptarget + prefix: /gpfs/neptune/spack-stack/llvm-21.1.0 + modules: + - llvm/21.1.0 + extra_attributes: + compilers: + c: /gpfs/neptune/spack-stack/llvm-21.1.0/bin/clang + cxx: /gpfs/neptune/spack-stack/llvm-21.1.0/bin/clang++ + fortran: /gpfs/neptune/spack-stack/llvm-21.1.0/bin/flang-new + extra_rpaths: + - /gpfs/neptune/spack-stack/llvm-21.1.0/lib + externals: + - spec: openmpi@4.1.8 + prefix: /gpfs/neptune/spack-stack/openmpi-4.1.8/llvm-21.1.0 + modules: + - openmpi/4.1.8 diff --git a/configs/sites/tier1/atlantis/packages_clang.yaml.notyetupdated b/configs/sites/tier1/atlantis/packages_clang.yaml.notyetupdated deleted file mode 100644 index 1c0be9013..000000000 --- a/configs/sites/tier1/atlantis/packages_clang.yaml.notyetupdated +++ /dev/null @@ -1,15 +0,0 @@ -packages: - all: - compiler:: [clang@20.1.5] - providers: - mpi:: [openmpi@5.0.6] - mpi: - buildable: False - openmpi: - buildable: False - externals: - - spec: openmpi@5.0.6 %clang@20.1.5 - modules: - - slurm - - openmpi/5.0.6 - diff --git a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml index 5566679c9..32f9360f9 100644 --- a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml +++ b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml @@ -1,13 +1,8 @@ packages: mpi: buildable: False - openmpi: - buildable: False - externals: - - spec: openmpi@4.1.8 %gcc@=13.4.0 - prefix: /gpfs/neptune/spack-stack/openmpi-4.1.8/gcc-13.4.0 - modules: - - openmpi/4.1.8 + require: + - "openmpi@4.1.8" gcc: externals: - spec: gcc@13.4.0 languages:='c,c++,fortran' @@ -23,3 +18,8 @@ packages: externals: - spec: gcc-runtime@13.4.0%gcc@13.4.0 prefix: /gpfs/neptune/spack-stack/gcc-13.4.0 + externals: + - spec: openmpi@4.1.8 + prefix: /gpfs/neptune/spack-stack/openmpi-4.1.8/gcc-13.4.0 + modules: + - openmpi/4.1.8 From 0396edf06825ca212cbb8909fa33a718ea79c00c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 26 Aug 2025 19:10:33 -0600 Subject: [PATCH 082/163] [skip ci] From 305bd92393bbf48183fc81e3f1b3356372b098db Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 27 Aug 2025 05:54:31 -0600 Subject: [PATCH 083/163] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index dc8e38807..194de92a2 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit dc8e38807900b193d8f826533db5ded1340d4f05 +Subproject commit 194de92a26cd7107f80483a4f4648fe305a2d824 From 0a9d2cee2d512795f9731f6e9f870ab2800ec265 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 27 Aug 2025 09:08:29 -0600 Subject: [PATCH 084/163] Update submodule pointer for repos/builtin --- .../{packages_aocc.yaml => packages_aocc.yaml.NOTYETUPDATED} | 0 ...apple-clang.yaml => packages_apple-clang.yaml.NOTYETUPDATED} | 0 .../{packages_intel.yaml => packages_intel.yaml.NOTYETUPDATED} | 0 repos/builtin | 2 +- 4 files changed, 1 insertion(+), 1 deletion(-) rename configs/common/{packages_aocc.yaml => packages_aocc.yaml.NOTYETUPDATED} (100%) rename configs/common/{packages_apple-clang.yaml => packages_apple-clang.yaml.NOTYETUPDATED} (100%) rename configs/common/{packages_intel.yaml => packages_intel.yaml.NOTYETUPDATED} (100%) diff --git a/configs/common/packages_aocc.yaml b/configs/common/packages_aocc.yaml.NOTYETUPDATED similarity index 100% rename from configs/common/packages_aocc.yaml rename to configs/common/packages_aocc.yaml.NOTYETUPDATED diff --git a/configs/common/packages_apple-clang.yaml b/configs/common/packages_apple-clang.yaml.NOTYETUPDATED similarity index 100% rename from configs/common/packages_apple-clang.yaml rename to configs/common/packages_apple-clang.yaml.NOTYETUPDATED diff --git a/configs/common/packages_intel.yaml b/configs/common/packages_intel.yaml.NOTYETUPDATED similarity index 100% rename from configs/common/packages_intel.yaml rename to configs/common/packages_intel.yaml.NOTYETUPDATED diff --git a/repos/builtin b/repos/builtin index 87e4b04a0..a9e61ea1f 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 87e4b04a04079e0c496179f8d0b6157fe707d825 +Subproject commit a9e61ea1f1416e17662f349ffaf41f23b1764d58 From 02a39575a528cb2769f7c0e62d23b831312ba3a1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 27 Aug 2025 09:09:38 -0600 Subject: [PATCH 085/163] Bug fix in configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml --- configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml b/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml index 4b769f5a7..71cb8348e 100644 --- a/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml +++ b/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml @@ -16,6 +16,7 @@ packages: fortran: /gpfs/neptune/spack-stack/llvm-21.1.0/bin/flang-new extra_rpaths: - /gpfs/neptune/spack-stack/llvm-21.1.0/lib + openmpi: externals: - spec: openmpi@4.1.8 prefix: /gpfs/neptune/spack-stack/openmpi-4.1.8/llvm-21.1.0 From 3987d4b9680930df40a8067b29fe6d61fba2647a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 27 Aug 2025 09:10:12 -0600 Subject: [PATCH 086/163] Update configs/common/packages*.yaml: use new spack v1 prefer syntax in updated yamls, turn off OpenMP for ip with clang and apple-clang --- configs/common/packages_aocc.yaml.NOTYETUPDATED | 1 + .../common/packages_apple-clang.yaml.NOTYETUPDATED | 1 + configs/common/packages_clang.yaml | 12 ++++++++++-- configs/common/packages_gcc.yaml | 11 +++++++++-- configs/common/packages_oneapi.yaml | 11 +++++++++-- 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/configs/common/packages_aocc.yaml.NOTYETUPDATED b/configs/common/packages_aocc.yaml.NOTYETUPDATED index f7240475a..0b09e12d2 100644 --- a/configs/common/packages_aocc.yaml.NOTYETUPDATED +++ b/configs/common/packages_aocc.yaml.NOTYETUPDATED @@ -17,3 +17,4 @@ packages: ip: require: - '%gcc' + - '^openblas' \ No newline at end of file diff --git a/configs/common/packages_apple-clang.yaml.NOTYETUPDATED b/configs/common/packages_apple-clang.yaml.NOTYETUPDATED index 9fa764bc2..7c1a45400 100644 --- a/configs/common/packages_apple-clang.yaml.NOTYETUPDATED +++ b/configs/common/packages_apple-clang.yaml.NOTYETUPDATED @@ -12,6 +12,7 @@ packages: - '~mkl' ip: require: + - '^openblas' - '~openmp' py-numpy: require: diff --git a/configs/common/packages_clang.yaml b/configs/common/packages_clang.yaml index 28baa48af..cfb952c6c 100644 --- a/configs/common/packages_clang.yaml +++ b/configs/common/packages_clang.yaml @@ -1,7 +1,14 @@ packages: - all: + c: + prefer: + - clang + cxx: prefer: - - '%clang' + - clang + fortran: + prefer: + - clang + all: providers: blas: [openblas] fftw-api: [fftw] @@ -12,6 +19,7 @@ packages: ip: require: - '^openblas' + - '~openmp' py-numpy: require: - '^openblas' diff --git a/configs/common/packages_gcc.yaml b/configs/common/packages_gcc.yaml index 82726d914..510e64433 100644 --- a/configs/common/packages_gcc.yaml +++ b/configs/common/packages_gcc.yaml @@ -1,7 +1,14 @@ packages: - all: + c: + prefer: + - gcc + cxx: prefer: - - '%gcc' + - gcc + fortran: + prefer: + - gcc + all: providers: blas: [openblas] fftw-api: [fftw] diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index c890fd2fd..ae9e17169 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -1,7 +1,14 @@ packages: - all: + c: + prefer: + - intel-oneapi-compilers + cxx: prefer: - - '%oneapi' + - intel-oneapi-compilers + fortran: + prefer: + - intel-oneapi-compilers + all: providers: blas: [intel-oneapi-mkl] fftw-api: [intel-oneapi-mkl] From 63eba2bf40a26cd46b502a6dbd2df8a655c067b3 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 27 Aug 2025 09:27:37 -0600 Subject: [PATCH 087/163] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 194de92a2..88ad07975 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 194de92a26cd7107f80483a4f4648fe305a2d824 +Subproject commit 88ad07975fae269cf47fc300458677453c400a31 From 77da0aa60978fa8cd3227795f3462c866f383788 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Aug 2025 05:16:58 -0600 Subject: [PATCH 088/163] Revert to old spack v023 require syntax to fix many duplicates wiht oneAPI environment, fix missing rpath entries for clang packages.yaml --- configs/common/packages.yaml | 11 ++++++----- configs/common/packages_clang.yaml | 11 ++--------- configs/common/packages_gcc.yaml | 11 ++--------- configs/common/packages_oneapi.yaml | 11 ++--------- .../sites/tier1/atlantis/packages_clang-21.1.0.yaml | 2 ++ configs/sites/tier2/bounty/packages_clang-20.1.5.yaml | 1 + 6 files changed, 15 insertions(+), 32 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 132cf006a..0d79d7d2e 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -378,11 +378,12 @@ packages: # To avoid duplicate packages py-urllib3: require: - - '@1.26.12' - ## To avoid duplicate packages - #py-versioneer: - # require: - # - '@0.28' + - '@1.26.20' + # OR - '@1.26.12' ? + # To avoid duplicate packages + py-versioneer: + require: + - '@0.29' qt: require: - '@5' diff --git a/configs/common/packages_clang.yaml b/configs/common/packages_clang.yaml index cfb952c6c..216cb7fe9 100644 --- a/configs/common/packages_clang.yaml +++ b/configs/common/packages_clang.yaml @@ -1,14 +1,7 @@ packages: - c: - prefer: - - clang - cxx: - prefer: - - clang - fortran: - prefer: - - clang all: + prefer: + - '%clang' providers: blas: [openblas] fftw-api: [fftw] diff --git a/configs/common/packages_gcc.yaml b/configs/common/packages_gcc.yaml index 510e64433..82726d914 100644 --- a/configs/common/packages_gcc.yaml +++ b/configs/common/packages_gcc.yaml @@ -1,14 +1,7 @@ packages: - c: - prefer: - - gcc - cxx: - prefer: - - gcc - fortran: - prefer: - - gcc all: + prefer: + - '%gcc' providers: blas: [openblas] fftw-api: [fftw] diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index ae9e17169..c890fd2fd 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -1,14 +1,7 @@ packages: - c: - prefer: - - intel-oneapi-compilers - cxx: - prefer: - - intel-oneapi-compilers - fortran: - prefer: - - intel-oneapi-compilers all: + prefer: + - '%oneapi' providers: blas: [intel-oneapi-mkl] fftw-api: [intel-oneapi-mkl] diff --git a/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml b/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml index 71cb8348e..2304ee10b 100644 --- a/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml +++ b/configs/sites/tier1/atlantis/packages_clang-21.1.0.yaml @@ -8,6 +8,7 @@ packages: - spec: llvm@21.1.0 +flang ~libomptarget prefix: /gpfs/neptune/spack-stack/llvm-21.1.0 modules: + - hwloc2/2.7.0 - llvm/21.1.0 extra_attributes: compilers: @@ -16,6 +17,7 @@ packages: fortran: /gpfs/neptune/spack-stack/llvm-21.1.0/bin/flang-new extra_rpaths: - /gpfs/neptune/spack-stack/llvm-21.1.0/lib + - /gpfs/neptune/spack-stack/llvm-21.1.0/lib/x86_64-unknown-linux-gnu openmpi: externals: - spec: openmpi@4.1.8 diff --git a/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml b/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml index df08bf982..4f2724b58 100644 --- a/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml +++ b/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml @@ -13,3 +13,4 @@ packages: fortran: /home/dom/prod/llvm-20.1.5/bin/flang-new extra_rpaths: - /home/dom/prod/llvm-20.1.5/lib + - /home/dom/prod/llvm-20.1.5/lib/x86_64-unknown-linux-gnu From 2a64a5ad8759090a1b2b6959ec6c917bb7d34a63 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Aug 2025 06:45:08 -0600 Subject: [PATCH 089/163] Update submodule pointer for builtin/repos --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index a9e61ea1f..bf4769d19 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit a9e61ea1f1416e17662f349ffaf41f23b1764d58 +Subproject commit bf4769d1951e4d375e22da1547d683be204b57b3 From f17470c20a8c8103a5eb7abe8fe2290f35c70465 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Aug 2025 19:50:48 -0600 Subject: [PATCH 090/163] Remove stack-python from .github/workflows/ubuntu-ci-x86_64-gnu.yaml, progress towards updating .github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml, update submodule pointer for repos/builtin --- .../TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml | 131 +++++++++--------- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 4 +- repos/builtin | 2 +- 3 files changed, 73 insertions(+), 64 deletions(-) diff --git a/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml index bdd999751..1ec2aad36 100644 --- a/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml @@ -32,7 +32,8 @@ jobs: - name: prepare-directories run: | - mkdir -p /home/ubuntu/spack-stack/build-cache/ + # DH* REVERT ME AFTER MERGE + mkdir -p /home/ubuntu/spack-stack/build-cache-new-spack-v1/ mkdir -p /home/ubuntu/spack-stack/source-cache/ - name: create-buildcache @@ -74,40 +75,53 @@ jobs: spack compiler find --scope system # For Intel oneAPI - echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo "- compiler:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " spec: oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " paths:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " cc: /opt/intel/oneapi/compiler/2024.2/bin/icx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " f77: /opt/intel/oneapi/compiler/2024.2/bin/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " fc: /opt/intel/oneapi/compiler/2024.2/bin/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " flags: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " operating_system: ubuntu22.04" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " target: x86_64" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " modules: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " environment: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " extra_rpaths: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - - # Need to find external Intel MPI and annotate with the - # correct compiler, no way to do that with spack commands. - echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - - # Add external Intel MKL and oneAPI runtime - echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " intel-oneapi-mkl:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " intel-oneapi-runtime:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + module use /opt/intel/oneapi/modulefiles + + cat << EOF >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2024.2.0 + prefix: /opt/intel/oneapi + modules: + - tbb/2021.13 + - compiler-rt/2024.2.0 + - compiler/2024.2.0 + extra_attributes: + compilers: + c: /opt/intel/oneapi/compiler/2024.2/bin/icx + cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx + fortran: /opt/intel/oneapi/compiler/2024.2/bin/ifort + intel-oneapi-runtime: + # LINK TO BUG REPORT HERE + #buildable: false + externals: + - spec: intel-oneapi-runtime@2024.2.0 + prefix: /opt/intel/oneapi + modules: + - tbb/2021.13 + - compiler-rt/2024.2.0 + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2021.13 + prefix: /opt/intel/oneapi + modules: + - tbb/2021.13 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2 + prefix: /opt/intel/oneapi + modules: + - mkl/2024.2 + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.13 + prefix: /opt/intel/oneapi + modules: + - mpi/2021.13 +EOF export SPACK_DISABLE_LOCAL_CONFIG=true unset SPACK_SYSTEM_CONFIG_PATH @@ -116,33 +130,25 @@ jobs: spack config add config:install_tree:padded_length:200 # Set compiler and MPI specs - spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.13]" - spack config add "packages:all:compiler:[oneapi@2024.2.0,gcc@11.4.0]" + spack config add "packages:mpi:buildable:False" + spack config add "packages:mpi:require:['intel-oneapi-mpi@2021.13']" + spack config add "packages:all:prefer:['%oneapi']" # Add additional variants for MET packages, different from config/common/packages.yaml spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" - # Switch providers for blas, lapack, fftw-api - spack config add "packages:all:providers:mpi:[intel-oneapi-mpi]" - spack config add "packages:all:providers:blas:[intel-oneapi-mkl]" - spack config add "packages:all:providers:fftw-api:[intel-oneapi-mkl]" - spack config add "packages:all:providers:lapack:[intel-oneapi-mkl]" - spack config add "packages:ectrans:require:'+mkl ~fftw'" - spack config add "packages:gsibec:require:'+mkl'" - spack config add "packages:py-numpy:require:['^intel-oneapi-mkl']" + ## Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) + #spack config add "packages:gmake:require:'@:4.2'" - # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) - spack config add "packages:gmake:require:'@:4.2'" - - # Remove wgrib2 from all virtual packages, since it doesn't build with oneAPI - sed -i 's/depends_on("wgrib2")/#depends_on("wgrib2")/g' `grep -lRie wgrib2 spack-ext/` + ## Remove wgrib2 from all virtual packages, since it doesn't build with oneAPI + #sed -i 's/depends_on("wgrib2")/#depends_on("wgrib2")/g' `grep -lRie wgrib2 spack-ext/` # Don't generate ecflow module when using external package spack config add "modules:default:tcl:exclude:[ecflow]" # Concretize and check for duplicates spack concretize 2>&1 | tee log.concretize.oneapi-2024.2.0-buildcache - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env # Add and update source cache spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/ @@ -151,8 +157,9 @@ jobs: # Add binary cache if requested if [ "$USE_BINARY_CACHE" = true ] ; then set +e - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ - spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) + # DH* REVERT ME AFTER MERGE + spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache-new-spack-v1/ + spack buildcache update-index local-binary set -e echo "Packages in spack binary cache:" spack buildcache list @@ -165,22 +172,22 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.base-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env + spack buildcache create -u local-binary base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-base-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u local-binary jedi-base-env # jedi-ufs-env echo "jedi-ufs-env ..." spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-ufs-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env + spack buildcache create -u local-binary jedi-ufs-env # the rest echo "unified-env ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.unified-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ + spack buildcache create -u local-binary # Check for libirc.so in shared libraries (returns 1=error if found) ${SPACK_STACK_DIR}/util/check_libirc.sh @@ -211,16 +218,17 @@ jobs: spack env activate ${ENVDIR} # Concretize and check for duplicates - spack concretize --force 2>&1 | tee log.concretize.oneapi-2024.2.0 - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl + spack concretize --force 2>&1 | tee log.concretize.${ENVNAME} + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env # Add binary cache back in - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ + # DH* REVERT ME AFTER MERGE + spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache-new-spack-v1/ echo "Packages in combined spack build caches:" spack buildcache list # Install from cache - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0.unified-env + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${ENVNAME}.all # Check shared libraries ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check @@ -246,7 +254,6 @@ jobs: module use ${ENVDIR}/install/modulefiles/Core module load stack-oneapi/2024.2.0 module load stack-intel-oneapi-mpi/2021.13 - module load stack-python/3.11.11 module available module load jedi-ufs-env diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 6aa88b82f..112a8b1f6 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -32,6 +32,7 @@ jobs: - name: prepare-directories run: | + # DH* REVERT ME AFTER MERGE mkdir -p /home/ubuntu/spack-stack/build-cache-new-spack-v1/ mkdir -p /home/ubuntu/spack-stack/source-cache/ @@ -110,6 +111,7 @@ jobs: # Add binary cache if requested if [ "$USE_BINARY_CACHE" = true ] ; then set +e + # DH* REVERT ME AFTER MERGE spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache-new-spack-v1/ spack buildcache update-index local-binary set -e @@ -175,6 +177,7 @@ jobs: ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env # Add binary cache back in + # DH* REVERT ME AFTER MERGE spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache-new-spack-v1/ echo "Packages in combined spack build caches:" spack buildcache list @@ -221,7 +224,6 @@ jobs: module use ${ENVDIR}/install/modulefiles/Core module load stack-gcc/11.4.0 module load stack-openmpi/5.0.8 - module load stack-python/3.11.11 module available module load jedi-ufs-env diff --git a/repos/builtin b/repos/builtin index bf4769d19..27bc9e51b 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit bf4769d1951e4d375e22da1547d683be204b57b3 +Subproject commit 27bc9e51bfd45064c965b9253f64f88640b34acd From 1449abf9fb234e4c965bf3e5559139cef44483be Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 29 Aug 2025 05:02:37 -0600 Subject: [PATCH 091/163] Bug fix in configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml and update of clang in util/nrl/batch_install.sh --- .github/workflows/{TMP_OFF => }/ubuntu-ci-x86_64-oneapi.yaml | 0 configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml | 1 + util/nrl/batch_install.sh | 2 +- 3 files changed, 2 insertions(+), 1 deletion(-) rename .github/workflows/{TMP_OFF => }/ubuntu-ci-x86_64-oneapi.yaml (100%) diff --git a/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml similarity index 100% rename from .github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml rename to .github/workflows/ubuntu-ci-x86_64-oneapi.yaml diff --git a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml index 32f9360f9..7a688d16f 100644 --- a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml +++ b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml @@ -18,6 +18,7 @@ packages: externals: - spec: gcc-runtime@13.4.0%gcc@13.4.0 prefix: /gpfs/neptune/spack-stack/gcc-13.4.0 + openmpi: externals: - spec: openmpi@4.1.8 prefix: /gpfs/neptune/spack-stack/openmpi-4.1.8/gcc-13.4.0 diff --git a/util/nrl/batch_install.sh b/util/nrl/batch_install.sh index 3e7c1ac64..1b10a9ff4 100755 --- a/util/nrl/batch_install.sh +++ b/util/nrl/batch_install.sh @@ -105,7 +105,7 @@ SPACK_STACK_BATCH_HOST=${SPACK_STACK_BATCH_HOST//[0-9]/} case ${SPACK_STACK_BATCH_HOST} in atlantis) - SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.1" "oneapi@=2025.0.3" "gcc@=13.4.0" "clang@=20.1.5") + SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.1" "oneapi@=2025.0.3" "gcc@=13.4.0" "clang@=21.1.0") SPACK_STACK_BATCH_TEMPLATES=("neptune-dev" "unified-dev" "cylc-dev") SPACK_STACK_MODULE_CHOICE="lmod" SPACK_STACK_BOOTSTRAP_MIRROR="/neptune_diagnostics/spack-stack/bootstrap-mirror" From 4ba45907f9f7705a2c465678c4b36659cccc96e2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 29 Aug 2025 05:03:19 -0600 Subject: [PATCH 092/163] Update .github/workflows/ubuntu-ci-x86_64-gnu.yaml and .github/workflows/ubuntu-ci-x86_64-oneapi.yaml --- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 9 +- .../workflows/ubuntu-ci-x86_64-oneapi.yaml | 251 ++++++++++-------- 2 files changed, 146 insertions(+), 114 deletions(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 112a8b1f6..b112e4dbe 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -11,6 +11,11 @@ defaults: run: shell: bash +env: + # DH* REVERT ME AFTER MERGE + BUILD_CACHE_PATH: /home/ubuntu/spack-stack/build-cache-new-spack-v1 + SOURCE_CACHE_PATH: /home/ubuntu/spack-stack/source-cache + jobs: ubuntu-ci-c6a-x86_64-gnu-build: runs-on: [ubuntu-ci-c6a-x86_64] @@ -33,8 +38,8 @@ jobs: - name: prepare-directories run: | # DH* REVERT ME AFTER MERGE - mkdir -p /home/ubuntu/spack-stack/build-cache-new-spack-v1/ - mkdir -p /home/ubuntu/spack-stack/source-cache/ + mkdir -p ${BUILD_CACHE_PATH} + mkdir -p ${SOURCE_CACHE_PATH} - name: create-buildcache run: | diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index 1ec2aad36..a55182ffc 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -11,6 +11,11 @@ defaults: run: shell: bash +env: + # DH* REVERT ME AFTER MERGE + BUILD_CACHE_PATH: /home/ubuntu/spack-stack/build-cache-new-spack-v1 + SOURCE_CACHE_PATH: /home/ubuntu/spack-stack/source-cache + jobs: ubuntu-ci-c6a-x86_64-oneapi-build: runs-on: [ubuntu-ci-c6a-x86_64] @@ -33,8 +38,8 @@ jobs: - name: prepare-directories run: | # DH* REVERT ME AFTER MERGE - mkdir -p /home/ubuntu/spack-stack/build-cache-new-spack-v1/ - mkdir -p /home/ubuntu/spack-stack/source-cache/ + mkdir -p ${BUILD_CACHE_PATH} + mkdir -p ${SOURCE_CACHE_PATH} - name: create-buildcache run: | @@ -51,33 +56,46 @@ jobs: # Set up spack-stack source ./setup.sh - export ENVNAME=ue-oneapi-2024.2.0-buildcache - export ENVDIR=$PWD/envs/${ENVNAME} - spack stack create env --site linux.default --template unified-dev --name ${ENVNAME} --compiler oneapi - spack env activate ${ENVDIR} - - unset SPACK_DISABLE_LOCAL_CONFIG - export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" - - # Find external packages - spack external find --scope system \ - --exclude bison --exclude openssl \ - --exclude curl --exclude python \ - --exclude meson --exclude gmake - spack external find --scope system grep - spack external find --scope system sed - spack external find --scope system perl - spack external find --scope system wget - spack external find --scope system texlive - spack external find --scope system mysql - # Find compilers - spack compiler find --scope system - - # For Intel oneAPI - module use /opt/intel/oneapi/modulefiles - - cat << EOF >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + declare -a TEMPLATES=("unified-dev" "skylab-dev" "neptune-dev") + for TEMPLATE in "${TEMPLATES[@]}"; do + if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then + export ENVNAME=ue-oneapi-2024.2.0-buildcache + elif [[ "${TEMPLATE}" == *"skylab-dev"* ]]; then + export ENVNAME=se-oneapi-2024.2.0-buildcache + elif [[ "${TEMPLATE}" == *"cylc-dev"* ]]; then + export ENVNAME=ce-oneapi-2024.2.0-buildcache + elif [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then + export ENVNAME=ne-oneapi-2024.2.0-buildcache + fi + echo "Creating environment ${ENVNAME} from template ${TEMPLATE}" + + export ENVDIR=$PWD/envs/${ENVNAME} + spack stack create env --site linux.default --template ${TEMPLATE} --name ${ENVNAME} --compiler oneapi + spack env activate ${ENVDIR} + + unset SPACK_DISABLE_LOCAL_CONFIG + export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" + + # Find external packages + spack external find --scope system \ + --exclude bison --exclude openssl \ + --exclude curl --exclude python \ + --exclude meson --exclude gmake + spack external find --scope system grep + spack external find --scope system sed + spack external find --scope system perl + spack external find --scope system wget + spack external find --scope system texlive + spack external find --scope system mysql + + # Find compilers + spack compiler find --scope system + + # For Intel oneAPI + module use /opt/intel/oneapi/modulefiles + + cat << EOF >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml intel-oneapi-compilers: externals: - spec: intel-oneapi-compilers@2024.2.0 @@ -123,88 +141,98 @@ jobs: - mpi/2021.13 EOF - export SPACK_DISABLE_LOCAL_CONFIG=true - unset SPACK_SYSTEM_CONFIG_PATH - - # For buildcaches - spack config add config:install_tree:padded_length:200 - - # Set compiler and MPI specs - spack config add "packages:mpi:buildable:False" - spack config add "packages:mpi:require:['intel-oneapi-mpi@2021.13']" - spack config add "packages:all:prefer:['%oneapi']" - - # Add additional variants for MET packages, different from config/common/packages.yaml - spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" - - ## Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) - #spack config add "packages:gmake:require:'@:4.2'" - - ## Remove wgrib2 from all virtual packages, since it doesn't build with oneAPI - #sed -i 's/depends_on("wgrib2")/#depends_on("wgrib2")/g' `grep -lRie wgrib2 spack-ext/` - - # Don't generate ecflow module when using external package - spack config add "modules:default:tcl:exclude:[ecflow]" - - # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.oneapi-2024.2.0-buildcache - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env - - # Add and update source cache - spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/ - spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/ - - # Add binary cache if requested - if [ "$USE_BINARY_CACHE" = true ] ; then - set +e - # DH* REVERT ME AFTER MERGE - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache-new-spack-v1/ - spack buildcache update-index local-binary - set -e - echo "Packages in spack binary cache:" - spack buildcache list - fi - - # Break installation up in pieces and create build caches in between - # This allows us to "spin up" builds that altogether take longer than - # six hours, and/or fail later in the build process. - - # base-env - echo "base-env ..." - spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.base-env - spack buildcache create -u local-binary base-env - - # jedi-base-env - echo "jedi-base-env ..." - spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-base-env - spack buildcache create -u local-binary jedi-base-env - - # jedi-ufs-env - echo "jedi-ufs-env ..." - spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-ufs-env - spack buildcache create -u local-binary jedi-ufs-env - - # the rest - echo "unified-env ..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.unified-env - spack buildcache create -u local-binary - - # Check for libirc.so in shared libraries (returns 1=error if found) - ${SPACK_STACK_DIR}/util/check_libirc.sh - - # Remove binary cache for next round of concretization - if [ "$USE_BINARY_CACHE" = true ] ; then - spack mirror rm local-binary - fi - - # Remove buildcache config settings - spack config remove config:install_tree:padded_length - - # Next steps: synchronize source and build cache to a central/combined mirror? - - # Cleanup - spack clean -a - spack env deactivate + export SPACK_DISABLE_LOCAL_CONFIG=true + unset SPACK_SYSTEM_CONFIG_PATH + + # For buildcaches + spack config add config:install_tree:padded_length:200 + + # Set compiler and MPI specs + spack config add "packages:mpi:buildable:False" + spack config add "packages:mpi:require:['intel-oneapi-mpi@2021.13']" + spack config add "packages:all:prefer:['%oneapi']" + + # Add additional variants for MET packages, different from config/common/packages.yaml + spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" + + ## Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) + #spack config add "packages:gmake:require:'@:4.2'" + + ## Remove wgrib2 from all virtual packages, since it doesn't build with oneAPI + #sed -i 's/depends_on("wgrib2")/#depends_on("wgrib2")/g' `grep -lRie wgrib2 spack-ext/` + + # Don't generate ecflow module when using external package + spack config add "modules:default:tcl:exclude:[ecflow]" + + # Concretize and check for duplicates + spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME} + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env + + # Add and update source cache + spack mirror add local-source file://${SOURCE_CACHE_PATH}/ + spack mirror create -a -d ${SOURCE_CACHE_PATH}/ + + # Add binary cache if requested + if [ "$USE_BINARY_CACHE" = true ] ; then + set +e + spack mirror add local-binary file://${BUILD_CACHE_PATH}/ + spack buildcache update-index local-binary + set -e + echo "Packages in spack binary cache:" + spack buildcache list + fi + + # Break installation up in pieces and create build caches in between + # This allows us to "spin up" builds that altogether take longer than + # six hours, and/or fail later in the build process. + + if [[ "${TEMPLATE}" == *"unified-dev"* || "${TEMPLATE}" == *"skylab-dev"* ]]; then + # base-env + echo "base-env ..." + spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env + spack buildcache create -u local-binary base-env + # jedi-base-env + echo "jedi-base-env ..." + spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.${ENVNAME}.jedi-base-env + spack buildcache create -u local-binary jedi-base-env + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.${ENVNAME}.jedi-ufs-env + spack buildcache create -u local-binary jedi-ufs-env + elif [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then + # base-env + echo "base-env ..." + spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env + spack buildcache create -u local-binary base-env + # neptune-env + echo "neptune-env ..." + spack install --fail-fast --source --no-check-signature neptune-env 2>&1 | tee log.install.${ENVNAME}.neptune-env + spack buildcache create -u local-binary neptune-env + fi + + # the rest + echo "${TEMPLATE} ..." + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${ENVNAME}.all + spack buildcache create -u local-binary + + # Check for libirc.so in shared libraries (returns 1=error if found) + ${SPACK_STACK_DIR}/util/check_libirc.sh + + # Remove binary cache for next round of concretization + if [ "$USE_BINARY_CACHE" = true ] ; then + spack mirror rm local-binary + fi + + # Remove buildcache config settings + spack config remove config:install_tree:padded_length + + # Next steps: synchronize source and build cache to a central/combined mirror? + + # Cleanup + spack clean -a + spack env deactivate + + done - name: create-env run: | @@ -222,8 +250,7 @@ EOF ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env # Add binary cache back in - # DH* REVERT ME AFTER MERGE - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache-new-spack-v1/ + spack mirror add local-binary file://${BUILD_CACHE_PATH}/ echo "Packages in combined spack build caches:" spack buildcache list From 7091e6acb17b4e0782bd6f49178c816f0bbd181a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 29 Aug 2025 05:49:59 -0600 Subject: [PATCH 093/163] Formatting/cleanup of .github/workflows/ubuntu-ci-x86_64-gnu.yaml --- .../TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml | 289 ++++++++++++++++++ .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 23 +- 2 files changed, 303 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml diff --git a/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml new file mode 100644 index 000000000..8a1607446 --- /dev/null +++ b/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml @@ -0,0 +1,289 @@ +name: ubuntu-ci-c6a-x86_64-oneapi-build +on: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + # DH* REVERT ME AFTER MERGE + BUILD_CACHE_PATH: /home/ubuntu/spack-stack/build-cache-new-spack-v1 + SOURCE_CACHE_PATH: /home/ubuntu/spack-stack/source-cache + +jobs: + ubuntu-ci-c6a-x86_64-oneapi-build: + runs-on: [ubuntu-ci-c6a-x86_64] + + steps: + - name: cleanup + run: | + pwd + ls -lart + set +e + find ./* -type d -exec chmod u+xw {} \; + set -e + rm -fr * + + - name: checkout + uses: actions/checkout@v4 + with: + submodules: true + + - name: prepare-directories + run: | + # DH* REVERT ME AFTER MERGE + mkdir -p ${BUILD_CACHE_PATH} + mkdir -p ${SOURCE_CACHE_PATH} + + - name: create-buildcache + run: | + # Get day of week to decide whether to use build caches or not + DOW=$(date +%u) + # Monday is 1 ... Sunday is 7 + if [[ $DOW == 7 ]]; then + export USE_BINARY_CACHE=false + echo "Ignore existing binary cache for creating buildcache environment" + else + export USE_BINARY_CACHE=true + echo "Use existing binary cache for creating buildcache environment" + fi + + # Set up spack-stack + source ./setup.sh + + declare -a TEMPLATES=("unified-dev" "skylab-dev" "neptune-dev") + for TEMPLATE in "${TEMPLATES[@]}"; do + if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then + export ENVNAME=ue-oneapi-2024.2.0-buildcache + elif [[ "${TEMPLATE}" == *"skylab-dev"* ]]; then + export ENVNAME=se-oneapi-2024.2.0-buildcache + elif [[ "${TEMPLATE}" == *"cylc-dev"* ]]; then + export ENVNAME=ce-oneapi-2024.2.0-buildcache + elif [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then + export ENVNAME=ne-oneapi-2024.2.0-buildcache + fi + echo "Creating environment ${ENVNAME} from template ${TEMPLATE}" + + export ENVDIR=$PWD/envs/${ENVNAME} + spack stack create env --site linux.default --template ${TEMPLATE} --name ${ENVNAME} --compiler oneapi + spack env activate ${ENVDIR} + + unset SPACK_DISABLE_LOCAL_CONFIG + export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" + + # Find external packages + spack external find --scope system \ + --exclude bison --exclude openssl \ + --exclude curl --exclude python \ + --exclude meson --exclude gmake + spack external find --scope system grep + spack external find --scope system sed + spack external find --scope system perl + spack external find --scope system wget + spack external find --scope system texlive + spack external find --scope system mysql + + # Find compilers + spack compiler find --scope system + + # For Intel oneAPI + module use /opt/intel/oneapi/modulefiles + + cat << EOF >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2024.2.0 + prefix: /opt/intel/oneapi + modules: + - tbb/2021.13 + - compiler-rt/2024.2.0 + - compiler/2024.2.0 + extra_attributes: + compilers: + c: /opt/intel/oneapi/compiler/2024.2/bin/icx + cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx + fortran: /opt/intel/oneapi/compiler/2024.2/bin/ifort + intel-oneapi-runtime: + # LINK TO BUG REPORT HERE + #buildable: false + externals: + - spec: intel-oneapi-runtime@2024.2.0 + prefix: /opt/intel/oneapi + modules: + - tbb/2021.13 + - compiler-rt/2024.2.0 + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2021.13 + prefix: /opt/intel/oneapi + modules: + - tbb/2021.13 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2 + prefix: /opt/intel/oneapi + modules: + - mkl/2024.2 + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.13 + prefix: /opt/intel/oneapi + modules: + - mpi/2021.13 + EOF + + export SPACK_DISABLE_LOCAL_CONFIG=true + unset SPACK_SYSTEM_CONFIG_PATH + + # For buildcaches + spack config add config:install_tree:padded_length:200 + + # Set compiler and MPI specs + spack config add "packages:mpi:buildable:False" + spack config add "packages:mpi:require:['intel-oneapi-mpi@2021.13']" + spack config add "packages:all:prefer:['%oneapi']" + + # Add additional variants for MET packages, different from config/common/packages.yaml + spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" + + ## Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) + #spack config add "packages:gmake:require:'@:4.2'" + + ## Remove wgrib2 from all virtual packages, since it doesn't build with oneAPI + #sed -i 's/depends_on("wgrib2")/#depends_on("wgrib2")/g' `grep -lRie wgrib2 spack-ext/` + + # Don't generate ecflow module when using external package + spack config add "modules:default:tcl:exclude:[ecflow]" + + # Concretize and check for duplicates + spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME} + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env + + # Add and update source cache + spack mirror add local-source file://${SOURCE_CACHE_PATH}/ + spack mirror create -a -d ${SOURCE_CACHE_PATH}/ + + # Add binary cache if requested + if [ "$USE_BINARY_CACHE" = true ] ; then + set +e + spack mirror add local-binary file://${BUILD_CACHE_PATH}/ + spack buildcache update-index local-binary + set -e + echo "Packages in spack binary cache:" + spack buildcache list + fi + + # Break installation up in pieces and create build caches in between + # This allows us to "spin up" builds that altogether take longer than + # six hours, and/or fail later in the build process. + + if [[ "${TEMPLATE}" == *"unified-dev"* || "${TEMPLATE}" == *"skylab-dev"* ]]; then + # base-env + echo "base-env ..." + spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env + spack buildcache create -u local-binary base-env + # jedi-base-env + echo "jedi-base-env ..." + spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.${ENVNAME}.jedi-base-env + spack buildcache create -u local-binary jedi-base-env + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.${ENVNAME}.jedi-ufs-env + spack buildcache create -u local-binary jedi-ufs-env + elif [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then + # base-env + echo "base-env ..." + spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env + spack buildcache create -u local-binary base-env + # neptune-env + echo "neptune-env ..." + spack install --fail-fast --source --no-check-signature neptune-env 2>&1 | tee log.install.${ENVNAME}.neptune-env + spack buildcache create -u local-binary neptune-env + fi + + # the rest + echo "${TEMPLATE} ..." + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${ENVNAME}.all + spack buildcache create -u local-binary + + # Check for libirc.so in shared libraries (returns 1=error if found) + ${SPACK_STACK_DIR}/util/check_libirc.sh + + # Remove binary cache for next round of concretization + if [ "$USE_BINARY_CACHE" = true ] ; then + spack mirror rm local-binary + fi + + # Remove buildcache config settings + spack config remove config:install_tree:padded_length + + # Next steps: synchronize source and build cache to a central/combined mirror? + + # Cleanup + spack clean -a + spack env deactivate + + done + + - name: create-env + run: | + # Set up spack-stack + source ./setup.sh + export BUILDCACHE_ENVNAME=ue-oneapi-2024.2.0-buildcache + export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} + export ENVNAME=ue-oneapi-2024.2.0 + export ENVDIR=$PWD/envs/${ENVNAME} + rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ + spack env activate ${ENVDIR} + + # Concretize and check for duplicates + spack concretize --force 2>&1 | tee log.concretize.${ENVNAME} + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env + + # Add binary cache back in + spack mirror add local-binary file://${BUILD_CACHE_PATH}/ + echo "Packages in combined spack build caches:" + spack buildcache list + + # Install from cache + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${ENVNAME}.all + + # Check shared libraries + ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check + + # Check for libirc.so in shared libraries (returns 1=error if found) + ${SPACK_STACK_DIR}/util/check_libirc.sh + + # Create modules + spack clean -a + spack module tcl refresh -y + spack stack setup-meta-modules + spack env deactivate + + - name: test-env + run: | + source /etc/profile.d/modules.sh + module use /home/ubuntu/spack-stack/modulefiles + + export ENVNAME=ue-oneapi-2024.2.0 + export ENVDIR=$PWD/envs/${ENVNAME} + ls -l ${ENVDIR}/install/modulefiles/Core + + module use ${ENVDIR}/install/modulefiles/Core + module load stack-oneapi/2024.2.0 + module load stack-intel-oneapi-mpi/2021.13 + module available + + module load jedi-ufs-env + module load ewok-env + module load soca-env + module list diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index b112e4dbe..d84f923e5 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -106,18 +106,17 @@ jobs: spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.${ENVNAME} + spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME} ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env # Add and update source cache - spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/ - spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/ + spack mirror add local-source file://${SOURCE_CACHE_PATH}/ + spack mirror create -a -d ${SOURCE_CACHE_PATH}/ # Add binary cache if requested if [ "$USE_BINARY_CACHE" = true ] ; then set +e - # DH* REVERT ME AFTER MERGE - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache-new-spack-v1/ + spack mirror add local-binary file://${BUILD_CACHE_PATH}/ spack buildcache update-index local-binary set -e echo "Packages in spack binary cache:" @@ -133,16 +132,23 @@ jobs: echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env spack buildcache create -u local-binary base-env - # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.${ENVNAME}.jedi-base-env spack buildcache create -u local-binary jedi-base-env - # jedi-ufs-env echo "jedi-ufs-env ..." spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.${ENVNAME}.jedi-ufs-env spack buildcache create -u local-binary jedi-ufs-env + elif [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then + # base-env + echo "base-env ..." + spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env + spack buildcache create -u local-binary base-env + # neptune-env + echo "neptune-env ..." + spack install --fail-fast --source --no-check-signature neptune-env 2>&1 | tee log.install.${ENVNAME}.neptune-env + spack buildcache create -u local-binary neptune-env fi # the rest @@ -182,8 +188,7 @@ jobs: ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env # Add binary cache back in - # DH* REVERT ME AFTER MERGE - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache-new-spack-v1/ + spack mirror add local-binary file://${BUILD_CACHE_PATH}/ echo "Packages in combined spack build caches:" spack buildcache list From ab79a0533911796aa22ca1b60a217bd8b8a66111 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 29 Aug 2025 07:39:18 -0600 Subject: [PATCH 094/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 27bc9e51b..2358b78d1 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 27bc9e51bfd45064c965b9253f64f88640b34acd +Subproject commit 2358b78d1bc5cb1a2e862ed74347ee416e4ac9d0 From 622b5c5666bfeb450d41654e84ed41423de0c2fb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 29 Aug 2025 07:39:53 -0600 Subject: [PATCH 095/163] Update compiler names in conflict statements in spack-ext/lib/jcsda-emc/spack-stack/stack/templates/{compiler,compiler.lua} --- spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler | 3 --- .../lib/jcsda-emc/spack-stack/stack/templates/compiler.lua | 3 --- 2 files changed, 6 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler index 0da4db8eb..65a42fd1e 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler @@ -12,10 +12,7 @@ conflict stack-gcc conflict stack-intel conflict stack-oneapi # new names as of July 2025 -# ??? # conflict stack-aocc -# ??? # conflict stack-apple-clang conflict stack-llvm -conflict stack-gcc conflict stack-intel-oneapi-compilers-classic conflict stack-intel-oneapi-compilers diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua index 1ff39dbc6..8c6339d29 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua @@ -16,10 +16,7 @@ conflict("stack-gcc") conflict("stack-intel") conflict("stack-oneapi") -- new names as of July 2025 --- ??? # conflict stack-aocc --- ??? # conflict stack-apple-clang conflict("stack-llvm") -conflict("stack-gcc") conflict("stack-intel-oneapi-compilers-classic") conflict("stack-intel-oneapi-compilers") From 006f8053010bf477a70bb367383e0cd4105a1a20 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 29 Aug 2025 08:46:00 -0600 Subject: [PATCH 096/163] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 88ad07975..83637b544 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 88ad07975fae269cf47fc300458677453c400a31 +Subproject commit 83637b544d6e251645aa44b793ac0bc6fc7ec802 From aa2359f8c98211d1725862e1e35ae03e09843180 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 29 Aug 2025 08:49:28 -0600 Subject: [PATCH 097/163] Find a better way to add oneAPI config in CI tests --- .github/workflows/ubuntu-ci-x86_64-oneapi.cfg | 43 +++++++++++++++++ .../workflows/ubuntu-ci-x86_64-oneapi.yaml | 46 +------------------ 2 files changed, 44 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/ubuntu-ci-x86_64-oneapi.cfg diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.cfg b/.github/workflows/ubuntu-ci-x86_64-oneapi.cfg new file mode 100644 index 000000000..d3e986def --- /dev/null +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.cfg @@ -0,0 +1,43 @@ + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2024.2.0 + prefix: /opt/intel/oneapi + modules: + - tbb/2021.13 + - compiler-rt/2024.2.0 + - compiler/2024.2.0 + extra_attributes: + compilers: + c: /opt/intel/oneapi/compiler/2024.2/bin/icx + cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx + fortran: /opt/intel/oneapi/compiler/2024.2/bin/ifort + intel-oneapi-runtime: + # LINK TO BUG REPORT HERE + #buildable: false + externals: + - spec: intel-oneapi-runtime@2024.2.0 + prefix: /opt/intel/oneapi + modules: + - tbb/2021.13 + - compiler-rt/2024.2.0 + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2021.13 + prefix: /opt/intel/oneapi + modules: + - tbb/2021.13 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2 + prefix: /opt/intel/oneapi + modules: + - mkl/2024.2 + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.13 + prefix: /opt/intel/oneapi + modules: + - mpi/2021.13 diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index a55182ffc..96f945238 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -95,51 +95,7 @@ jobs: # For Intel oneAPI module use /opt/intel/oneapi/modulefiles - cat << EOF >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - intel-oneapi-compilers: - externals: - - spec: intel-oneapi-compilers@2024.2.0 - prefix: /opt/intel/oneapi - modules: - - tbb/2021.13 - - compiler-rt/2024.2.0 - - compiler/2024.2.0 - extra_attributes: - compilers: - c: /opt/intel/oneapi/compiler/2024.2/bin/icx - cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx - fortran: /opt/intel/oneapi/compiler/2024.2/bin/ifort - intel-oneapi-runtime: - # LINK TO BUG REPORT HERE - #buildable: false - externals: - - spec: intel-oneapi-runtime@2024.2.0 - prefix: /opt/intel/oneapi - modules: - - tbb/2021.13 - - compiler-rt/2024.2.0 - intel-oneapi-tbb: - buildable: false - externals: - - spec: intel-oneapi-tbb@2021.13 - prefix: /opt/intel/oneapi - modules: - - tbb/2021.13 - intel-oneapi-mkl: - buildable: false - externals: - - spec: intel-oneapi-mkl@2024.2 - prefix: /opt/intel/oneapi - modules: - - mkl/2024.2 - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.13 - prefix: /opt/intel/oneapi - modules: - - mpi/2021.13 -EOF + cat .github/workflows/ubuntu-ci-x86_64-oneapi.cfg >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml export SPACK_DISABLE_LOCAL_CONFIG=true unset SPACK_SYSTEM_CONFIG_PATH From 1ee6032ab04772f88f5912cf9faa156638e5ee55 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 29 Aug 2025 09:35:31 -0600 Subject: [PATCH 098/163] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 83637b544..f1c3b7a10 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 83637b544d6e251645aa44b793ac0bc6fc7ec802 +Subproject commit f1c3b7a1085e098986a2832ee19b9eea4baecc71 From 3d1e6c057939961c4a92afdfa41552d77d55fe37 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 29 Aug 2025 09:57:52 -0600 Subject: [PATCH 099/163] Fix .github/workflows/ubuntu-ci-x86_64-oneapi.yaml, temporarily turn off .github/workflows/ubuntu-ci-x86_64-gnu.yaml --- ...buntu-ci-x86_64-gnu.yaml => ubuntu-ci-x86_64-gnu.yaml.tmpoff} | 0 .github/workflows/ubuntu-ci-x86_64-oneapi.yaml | 1 + 2 files changed, 1 insertion(+) rename .github/workflows/{ubuntu-ci-x86_64-gnu.yaml => ubuntu-ci-x86_64-gnu.yaml.tmpoff} (100%) diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml.tmpoff similarity index 100% rename from .github/workflows/ubuntu-ci-x86_64-gnu.yaml rename to .github/workflows/ubuntu-ci-x86_64-gnu.yaml.tmpoff diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index 96f945238..af76a7e41 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -93,6 +93,7 @@ jobs: spack compiler find --scope system # For Intel oneAPI + source /usr/share/modules/init/bash module use /opt/intel/oneapi/modulefiles cat .github/workflows/ubuntu-ci-x86_64-oneapi.cfg >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml From 3662b35814e7a3cf576d934062b0677aa6ded0dd Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Sep 2025 05:44:29 -0600 Subject: [PATCH 100/163] Add .github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml and fix .github/workflows/ubuntu-ci-x86_64-oneapi.yaml --- .../TMP_OFF/ubuntu-ci-x86_64-oneapi-ifx.yaml | 255 ------------------ .../workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg | 43 +++ ....yaml => ubuntu-ci-x86_64-oneapi-ifx.yaml} | 71 +---- .../workflows/ubuntu-ci-x86_64-oneapi.yaml | 6 +- 4 files changed, 60 insertions(+), 315 deletions(-) delete mode 100644 .github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi-ifx.yaml create mode 100644 .github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg rename .github/workflows/{TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml => ubuntu-ci-x86_64-oneapi-ifx.yaml} (83%) diff --git a/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi-ifx.yaml b/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi-ifx.yaml deleted file mode 100644 index 033934671..000000000 --- a/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi-ifx.yaml +++ /dev/null @@ -1,255 +0,0 @@ -name: ubuntu-ci-c6a-x86_64-oneapi-ifx-build -on: - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - ubuntu-ci-c6a-x86_64-oneapi-ifx-build: - runs-on: [ubuntu-ci-c6a-x86_64] - - steps: - - name: cleanup - run: | - pwd - ls -lart - set +e - find ./* -type d -exec chmod u+xw {} \; - set -e - rm -fr * - - - name: checkout - uses: actions/checkout@v4 - with: - submodules: true - - - name: prepare-directories - run: | - mkdir -p /home/ubuntu/spack-stack/build-cache/ - mkdir -p /home/ubuntu/spack-stack/source-cache/ - - - name: create-buildcache - run: | - # Get day of week to decide whether to use build caches or not - DOW=$(date +%u) - # Monday is 1 ... Sunday is 7 - if [[ $DOW == 7 ]]; then - export USE_BINARY_CACHE=false - echo "Ignore existing binary cache for creating buildcache environment" - else - export USE_BINARY_CACHE=true - echo "Use existing binary cache for creating buildcache environment" - fi - - # Set up spack-stack - source ./setup.sh - export ENVNAME=ue-oneifx-2024.2.0-buildcache - export ENVDIR=$PWD/envs/${ENVNAME} - spack stack create env --site linux.default --template unified-dev --name ${ENVNAME} --compiler oneapi - spack env activate ${ENVDIR} - - unset SPACK_DISABLE_LOCAL_CONFIG - export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" - - # Find external packages - spack external find --scope system \ - --exclude bison --exclude openssl \ - --exclude curl --exclude python \ - --exclude meson --exclude gmake - spack external find --scope system grep - spack external find --scope system sed - spack external find --scope system perl - spack external find --scope system wget - spack external find --scope system texlive - spack external find --scope system mysql - - # Find compilers - spack compiler find --scope system - - # For Intel oneAPI - echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo "- compiler:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " spec: oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " paths:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " cc: /opt/intel/oneapi/compiler/2024.2/bin/icx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " f77: /opt/intel/oneapi/compiler/2024.2/bin/ifx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " fc: /opt/intel/oneapi/compiler/2024.2/bin/ifx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " flags: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " operating_system: ubuntu22.04" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " target: x86_64" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " modules: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " environment: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " extra_rpaths: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - - # Need to find external Intel MPI and annotate with the - # correct compiler, no way to do that with spack commands. - echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - - # Add external Intel MKL and oneAPI runtime - echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " intel-oneapi-mkl:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " intel-oneapi-runtime:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - - export SPACK_DISABLE_LOCAL_CONFIG=true - unset SPACK_SYSTEM_CONFIG_PATH - - # For buildcaches - spack config add config:install_tree:padded_length:200 - - # Set compiler and MPI specs - spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.13]" - spack config add "packages:all:compiler:[oneapi@2024.2.0,gcc@11.4.0]" - - # Add additional variants for MET packages, different from config/common/packages.yaml - spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" - - # Switch providers for blas, lapack, fftw-api - spack config add "packages:all:providers:mpi:[intel-oneapi-mpi]" - spack config add "packages:all:providers:blas:[intel-oneapi-mkl]" - spack config add "packages:all:providers:fftw-api:[intel-oneapi-mkl]" - spack config add "packages:all:providers:lapack:[intel-oneapi-mkl]" - spack config add "packages:ectrans:require:'+mkl ~fftw'" - spack config add "packages:gsibec:require:'+mkl'" - spack config add "packages:py-numpy:require:['^intel-oneapi-mkl']" - - # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) - spack config add "packages:gmake:require:'@:4.2'" - - # Remove wgrib2 from all virtual packages, since it doesn't build with oneAPI - sed -i 's/depends_on("wgrib2")/#depends_on("wgrib2")/g' `grep -lRie wgrib2 spack-ext/` - - # Don't generate ecflow module when using external package - spack config add "modules:default:tcl:exclude:[ecflow]" - - # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.oneifx-2024.2.0-buildcache - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl - - # Add and update source cache - spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/ - spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/ - - # Add binary cache if requested - if [ "$USE_BINARY_CACHE" = true ] ; then - set +e - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ - spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) - set -e - echo "Packages in spack binary cache:" - spack buildcache list - fi - - # Break installation up in pieces and create build caches in between - # This allows us to "spin up" builds that altogether take longer than - # six hours, and/or fail later in the build process. - - # base-env - echo "base-env ..." - spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneifx-2024.2.0-buildcache.base-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env - - # jedi-base-env - echo "jedi-base-env ..." - spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneifx-2024.2.0-buildcache.jedi-base-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env - - # jedi-ufs-env - echo "jedi-ufs-env ..." - spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneifx-2024.2.0-buildcache.jedi-ufs-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env - - # the rest - echo "unified-env ..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneifx-2024.2.0-buildcache.unified-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ - - # Check for libirc.so in shared libraries (returns 1=error if found) - ${SPACK_STACK_DIR}/util/check_libirc.sh - - # Remove binary cache for next round of concretization - if [ "$USE_BINARY_CACHE" = true ] ; then - spack mirror rm local-binary - fi - - # Remove buildcache config settings - spack config remove config:install_tree:padded_length - - # Next steps: synchronize source and build cache to a central/combined mirror? - - # Cleanup - spack clean -a - spack env deactivate - - - name: create-env - run: | - # Set up spack-stack - source ./setup.sh - export BUILDCACHE_ENVNAME=ue-oneifx-2024.2.0-buildcache - export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} - export ENVNAME=ue-oneifx-2024.2.0 - export ENVDIR=$PWD/envs/${ENVNAME} - rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ - spack env activate ${ENVDIR} - - # Concretize and check for duplicates - spack concretize --force 2>&1 | tee log.concretize.oneifx-2024.2.0 - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl - - # Add binary cache back in - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ - echo "Packages in combined spack build caches:" - spack buildcache list - - # Install from cache - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneifx-2024.2.0.unified-env - - # Check shared libraries - ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check - - # Check for libirc.so in shared libraries (returns 1=error if found) - ${SPACK_STACK_DIR}/util/check_libirc.sh - - # Create modules - spack clean -a - spack module tcl refresh -y - spack stack setup-meta-modules - spack env deactivate - - - name: test-env - run: | - source /etc/profile.d/modules.sh - module use /home/ubuntu/spack-stack/modulefiles - - export ENVNAME=ue-oneifx-2024.2.0 - export ENVDIR=$PWD/envs/${ENVNAME} - ls -l ${ENVDIR}/install/modulefiles/Core - - module use ${ENVDIR}/install/modulefiles/Core - module load stack-oneapi/2024.2.0 - module load stack-intel-oneapi-mpi/2021.13 - module load stack-python/3.11.11 - module available - - module load jedi-ufs-env - module load ewok-env - module load soca-env - module list diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg new file mode 100644 index 000000000..d1d15f9da --- /dev/null +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg @@ -0,0 +1,43 @@ + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2024.2.0 + prefix: /opt/intel/oneapi + modules: + - tbb/2021.13 + - compiler-rt/2024.2.0 + - compiler/2024.2.0 + extra_attributes: + compilers: + c: /opt/intel/oneapi/compiler/2024.2/bin/icx + cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx + fortran: /opt/intel/oneapi/compiler/2024.2/bin/ifx + intel-oneapi-runtime: + # LINK TO BUG REPORT HERE + #buildable: false + externals: + - spec: intel-oneapi-runtime@2024.2.0 + prefix: /opt/intel/oneapi + modules: + - tbb/2021.13 + - compiler-rt/2024.2.0 + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2021.13 + prefix: /opt/intel/oneapi + modules: + - tbb/2021.13 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2 + prefix: /opt/intel/oneapi + modules: + - mkl/2024.2 + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.13 + prefix: /opt/intel/oneapi + modules: + - mpi/2021.13 diff --git a/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml similarity index 83% rename from .github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml rename to .github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml index 8a1607446..748424204 100644 --- a/.github/workflows/TMP_OFF/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml @@ -1,4 +1,4 @@ -name: ubuntu-ci-c6a-x86_64-oneapi-build +name: ubuntu-ci-c6a-x86_64-oneapi-ifx-build on: pull_request: workflow_dispatch: @@ -17,7 +17,7 @@ env: SOURCE_CACHE_PATH: /home/ubuntu/spack-stack/source-cache jobs: - ubuntu-ci-c6a-x86_64-oneapi-build: + ubuntu-ci-c6a-x86_64-oneapi-ifx-build: runs-on: [ubuntu-ci-c6a-x86_64] steps: @@ -60,13 +60,13 @@ jobs: declare -a TEMPLATES=("unified-dev" "skylab-dev" "neptune-dev") for TEMPLATE in "${TEMPLATES[@]}"; do if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then - export ENVNAME=ue-oneapi-2024.2.0-buildcache + export ENVNAME=ue-oneifx-2024.2.0-buildcache elif [[ "${TEMPLATE}" == *"skylab-dev"* ]]; then - export ENVNAME=se-oneapi-2024.2.0-buildcache + export ENVNAME=se-oneifx-2024.2.0-buildcache elif [[ "${TEMPLATE}" == *"cylc-dev"* ]]; then - export ENVNAME=ce-oneapi-2024.2.0-buildcache + export ENVNAME=ce-oneifx-2024.2.0-buildcache elif [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then - export ENVNAME=ne-oneapi-2024.2.0-buildcache + export ENVNAME=ne-oneifx-2024.2.0-buildcache fi echo "Creating environment ${ENVNAME} from template ${TEMPLATE}" @@ -93,53 +93,10 @@ jobs: spack compiler find --scope system # For Intel oneAPI + source /usr/share/modules/init/bash module use /opt/intel/oneapi/modulefiles - cat << EOF >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - intel-oneapi-compilers: - externals: - - spec: intel-oneapi-compilers@2024.2.0 - prefix: /opt/intel/oneapi - modules: - - tbb/2021.13 - - compiler-rt/2024.2.0 - - compiler/2024.2.0 - extra_attributes: - compilers: - c: /opt/intel/oneapi/compiler/2024.2/bin/icx - cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx - fortran: /opt/intel/oneapi/compiler/2024.2/bin/ifort - intel-oneapi-runtime: - # LINK TO BUG REPORT HERE - #buildable: false - externals: - - spec: intel-oneapi-runtime@2024.2.0 - prefix: /opt/intel/oneapi - modules: - - tbb/2021.13 - - compiler-rt/2024.2.0 - intel-oneapi-tbb: - buildable: false - externals: - - spec: intel-oneapi-tbb@2021.13 - prefix: /opt/intel/oneapi - modules: - - tbb/2021.13 - intel-oneapi-mkl: - buildable: false - externals: - - spec: intel-oneapi-mkl@2024.2 - prefix: /opt/intel/oneapi - modules: - - mkl/2024.2 - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.13 - prefix: /opt/intel/oneapi - modules: - - mpi/2021.13 - EOF + cat .github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml export SPACK_DISABLE_LOCAL_CONFIG=true unset SPACK_SYSTEM_CONFIG_PATH @@ -204,10 +161,10 @@ jobs: echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env spack buildcache create -u local-binary base-env - # neptune-env + # neptune-python-env echo "neptune-env ..." - spack install --fail-fast --source --no-check-signature neptune-env 2>&1 | tee log.install.${ENVNAME}.neptune-env - spack buildcache create -u local-binary neptune-env + spack install --fail-fast --source --no-check-signature neptune-python-env 2>&1 | tee log.install.${ENVNAME}.neptune-env + spack buildcache create -u local-binary neptune-python-env fi # the rest @@ -238,9 +195,9 @@ jobs: run: | # Set up spack-stack source ./setup.sh - export BUILDCACHE_ENVNAME=ue-oneapi-2024.2.0-buildcache + export BUILDCACHE_ENVNAME=ue-oneifx-2024.2.0-buildcache export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} - export ENVNAME=ue-oneapi-2024.2.0 + export ENVNAME=ue-oneifx-2024.2.0 export ENVDIR=$PWD/envs/${ENVNAME} rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ spack env activate ${ENVDIR} @@ -274,7 +231,7 @@ jobs: source /etc/profile.d/modules.sh module use /home/ubuntu/spack-stack/modulefiles - export ENVNAME=ue-oneapi-2024.2.0 + export ENVNAME=ue-oneifx-2024.2.0 export ENVDIR=$PWD/envs/${ENVNAME} ls -l ${ENVDIR}/install/modulefiles/Core diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index af76a7e41..cffc516ff 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -161,10 +161,10 @@ jobs: echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env spack buildcache create -u local-binary base-env - # neptune-env + # neptune-python-env echo "neptune-env ..." - spack install --fail-fast --source --no-check-signature neptune-env 2>&1 | tee log.install.${ENVNAME}.neptune-env - spack buildcache create -u local-binary neptune-env + spack install --fail-fast --source --no-check-signature neptune-python-env 2>&1 | tee log.install.${ENVNAME}.neptune-env + spack buildcache create -u local-binary neptune-python-env fi # the rest From 828fca57f769bc8507f4e471c9ee56d3354bd13f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Sep 2025 12:47:29 -0600 Subject: [PATCH 101/163] Update .github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml and .github/workflows/ubuntu-ci-x86_64-oneapi.yaml, remove .github/workflows/TMP_OFF/macos-ci-aarch64.yaml entirely --- .../workflows/TMP_OFF/macos-ci-aarch64.yaml | 221 ------------------ .../ubuntu-ci-x86_64-oneapi-ifx.yaml | 6 +- .../workflows/ubuntu-ci-x86_64-oneapi.yaml | 6 +- 3 files changed, 10 insertions(+), 223 deletions(-) delete mode 100644 .github/workflows/TMP_OFF/macos-ci-aarch64.yaml diff --git a/.github/workflows/TMP_OFF/macos-ci-aarch64.yaml b/.github/workflows/TMP_OFF/macos-ci-aarch64.yaml deleted file mode 100644 index 7b6dbd594..000000000 --- a/.github/workflows/TMP_OFF/macos-ci-aarch64.yaml +++ /dev/null @@ -1,221 +0,0 @@ -name: macos-ci-aarch64-build -on: - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - macos-ci-aarch64-build: - runs-on: [macos-ci-aarch64] - - steps: - - name: cleanup - run: | - pwd - ls -lart - set +e - find ./* -type d -exec chmod u+xw {} \; - set -e - rm -fr * - - - name: checkout - uses: actions/checkout@v4 - with: - submodules: true - - - name: prepare-directories - run: | - mkdir -p /Users/ec2-user/spack-stack/build-cache - mkdir -p /Users/ec2-user/spack-stack/source-cache - - - name: create-buildcache - run: | - # Get day of week to decide whether to use build caches or not - DOW=$(date +%u) - # Monday is 1 ... Sunday is 7 - if [[ $DOW == 7 ]]; then - export USE_BINARY_CACHE=false - echo "Ignore existing binary cache for creating buildcache environment" - else - export USE_BINARY_CACHE=true - echo "Use existing binary cache for creating buildcache environment" - fi - - # Set up homebrew and lmod support - eval "$(/opt/homebrew/bin/brew shellenv)" - source /opt/homebrew/opt/lmod/init/profile - - # Set up spack-stack - source ./setup.sh - # Important! - export SPACK_PYTHON=/usr/bin/python3 - spack clean -ab - spack bootstrap now - # - export ENVNAME=ue-apple-clang-14.0.3-buildcache - export ENVDIR=$PWD/envs/${ENVNAME} - spack stack create env --site macos.default --template unified-dev --name ${ENVNAME} --compiler apple-clang - spack env activate ${ENVDIR} - - unset SPACK_DISABLE_LOCAL_CONFIG - export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" - - # Find external packages - spack external find --scope system \ - --exclude bison --exclude openssl \ - --exclude curl --exclude python - spack external find --scope system grep - spack external find --scope system perl - spack external find --scope system libiconv - spack external find --scope system wget - PATH="/opt/homebrew/opt/curl/bin:$PATH" \ - spack external find --scope system curl - PATH="/opt/homebrew/opt/qt5/bin:$PATH" \ - spack external find --scope system qt - spack external find --scope system texlive - spack external find --scope system mysql - - # Find compilers - spack compiler find --scope system - - export SPACK_DISABLE_LOCAL_CONFIG=true - unset SPACK_SYSTEM_CONFIG_PATH - - # For buildcaches - spack config add config:install_tree:padded_length:200 - - # Set compiler and MPI specs - spack config add "packages:all:providers:mpi:[openmpi@5.0.5]" - spack config add "packages:all:compiler:[apple-clang@14.0.3]" - - # Add additional variants for MET packages, different from config/common/packages.yaml - # DH* 20240513 - avoid hdf-eos2 until https://github.com/spack/spack/issues/44168 is resolved - #spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" - spack config add "packages:met:variants:+python +grib2 +graphics" - # *DH - - # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3-buildcache - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl - - # Add and update source cache - spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/ - spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/ - - # Add binary cache if requested - if [ "$USE_BINARY_CACHE" = true ] ; then - set +e - spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ - spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) - set -e - echo "Packages in spack binary cache:" - spack buildcache list - fi - - # Break installation up in pieces and create build caches in between - # This allows us to "spin up" builds that altogether take longer than - # six hours, and/or fail later in the build process. - - # base-env - echo "base-env ..." - spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env - spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ base-env - - # jedi-base-env - echo "jedi-base-env ..." - spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-base-env - spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env - - # jedi-ufs-env - echo "jedi-ufs-env ..." - spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-ufs-env - spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ jedi-ufs-env - - # the rest - echo "unified-env ..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env - spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ - - # Remove binary cache for next round of concretization - if [ "$USE_BINARY_CACHE" = true ] ; then - spack mirror rm local-binary - fi - - # Remove buildcache config settings - spack config remove config:install_tree:padded_length - - # Next steps: synchronize source and build cache to a central/combined mirror? - - # Cleanup - spack clean -a - spack env deactivate - - - name: create-env - run: | - # Set up spack-stack - source ./setup.sh - # Important! - export SPACK_PYTHON=/usr/bin/python3 - export BUILDCACHE_ENVNAME=ue-apple-clang-14.0.3-buildcache - export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} - export ENVNAME=ue-apple-clang-14.0.3 - export ENVDIR=$PWD/envs/${ENVNAME} - rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ - spack env activate ${ENVDIR} - - # Concretize and check for duplicates - spack concretize --force 2>&1 | tee log.concretize.apple-clang-14.0.3 - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl - - # Add binary cache back in - spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ - echo "Packages in combined spack build caches:" - spack buildcache list - - # Install from cache - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env - - # Create modules - spack clean -a - spack module lmod refresh -y - spack stack setup-meta-modules - spack env deactivate - - # Test environment chaining - echo "Test environment chaining" - spack stack create env --name chaintest --template empty --site macos.default --upstream ${ENVDIR}/install --compiler apple-clang - # Retain config from upstream so we don't have to rebuild: - cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/. - spack env activate ${PWD}/envs/chaintest - spack add nccmp@1.9.0.1%apple-clang - spack concretize | tee envs/chaintest/log.concretize - unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l) - if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi - spack env deactivate - - - name: test-env - run: | - eval "$(/opt/homebrew/bin/brew shellenv)" - source /opt/homebrew/opt/lmod/init/profile - - export ENVNAME=ue-apple-clang-14.0.3 - export ENVDIR=$PWD/envs/${ENVNAME} - ls -l ${ENVDIR}/install/modulefiles/Core - - module use ${ENVDIR}/install/modulefiles/Core - module load stack-apple-clang/14.0.3 - module load stack-openmpi/5.0.6 - module load stack-python/3.11.11 - module available - - module load jedi-ufs-env - module load ewok-env - module load soca-env - module list diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml index 748424204..5647c7fb7 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml @@ -93,7 +93,7 @@ jobs: spack compiler find --scope system # For Intel oneAPI - source /usr/share/modules/init/bash + source /etc/profile.d/modules.sh module use /opt/intel/oneapi/modulefiles cat .github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml @@ -214,6 +214,9 @@ jobs: # Install from cache spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${ENVNAME}.all + # For spack-built Intel oneAPI runtime to find libintlc.so.5 etc. + export LD_LIBRARY_PATH="/opt/intel/oneapi/compiler/2024.2/lib:$LD_LIBRARY_PATH" + # Check shared libraries ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check @@ -229,6 +232,7 @@ jobs: - name: test-env run: | source /etc/profile.d/modules.sh + module use /opt/intel/oneapi/modulefiles module use /home/ubuntu/spack-stack/modulefiles export ENVNAME=ue-oneifx-2024.2.0 diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index cffc516ff..06cda78f1 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -93,7 +93,7 @@ jobs: spack compiler find --scope system # For Intel oneAPI - source /usr/share/modules/init/bash + source /etc/profile.d/modules.sh module use /opt/intel/oneapi/modulefiles cat .github/workflows/ubuntu-ci-x86_64-oneapi.cfg >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml @@ -214,6 +214,9 @@ jobs: # Install from cache spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${ENVNAME}.all + # For spack-built Intel oneAPI runtime to find libintlc.so.5 etc. + export LD_LIBRARY_PATH="/opt/intel/oneapi/compiler/2024.2/lib:$LD_LIBRARY_PATH" + # Check shared libraries ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check @@ -229,6 +232,7 @@ jobs: - name: test-env run: | source /etc/profile.d/modules.sh + module use /opt/intel/oneapi/modulefiles module use /home/ubuntu/spack-stack/modulefiles export ENVNAME=ue-oneapi-2024.2.0 From 159b8b3b7abd53bb0feb6b9feb903542d716d21d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Sep 2025 14:07:06 -0600 Subject: [PATCH 102/163] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index f1c3b7a10..b94ce1d7a 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit f1c3b7a1085e098986a2832ee19b9eea4baecc71 +Subproject commit b94ce1d7a26c6935d1b6f15b8febf59deed39a84 From 2c7fa9ba24d3402ab1d86a24c57eaf71589b5ab1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Sep 2025 19:56:24 -0600 Subject: [PATCH 103/163] Bug fixes in .github/workflows/ubuntu-ci-x86_64-oneapi*.yaml: update stack-oneapi to stack-intel-oneapi-compilers --- .github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml | 2 +- .github/workflows/ubuntu-ci-x86_64-oneapi.yaml | 2 +- spack | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml index 5647c7fb7..c8512719d 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml @@ -240,7 +240,7 @@ jobs: ls -l ${ENVDIR}/install/modulefiles/Core module use ${ENVDIR}/install/modulefiles/Core - module load stack-oneapi/2024.2.0 + module load stack-intel-oneapi-compilers/2024.2.0 module load stack-intel-oneapi-mpi/2021.13 module available diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index 06cda78f1..ab6943cb0 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -240,7 +240,7 @@ jobs: ls -l ${ENVDIR}/install/modulefiles/Core module use ${ENVDIR}/install/modulefiles/Core - module load stack-oneapi/2024.2.0 + module load stack-intel-oneapi-compilers/2024.2.0 module load stack-intel-oneapi-mpi/2021.13 module available diff --git a/spack b/spack index b94ce1d7a..79385e545 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit b94ce1d7a26c6935d1b6f15b8febf59deed39a84 +Subproject commit 79385e545912dd05008d75bfaca0200c8313b399 From f2347800f47160ecbb7cd8ff99a83d6069030b9b Mon Sep 17 00:00:00 2001 From: Rick Grubin Date: Tue, 2 Sep 2025 20:00:00 -0600 Subject: [PATCH 104/163] gaea-c6: fix errors in site-specific compilers.yaml and packages.yaml (#1764) --- configs/sites/tier1/gaea-c6/compilers.yaml | 2 +- configs/sites/tier1/gaea-c6/packages.yaml | 5 ----- spack | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/configs/sites/tier1/gaea-c6/compilers.yaml b/configs/sites/tier1/gaea-c6/compilers.yaml index 86025058a..0bd9c3a31 100644 --- a/configs/sites/tier1/gaea-c6/compilers.yaml +++ b/configs/sites/tier1/gaea-c6/compilers.yaml @@ -69,7 +69,7 @@ compilers: - libfabric/1.20.1 - cray-libsci/24.11.0 - libfabric/1.20.1 - environment: {} + environment: prepend_path: LD_LIBRARY_PATH: '/opt/cray/libfabric/1.20.1/lib64:/opt/cray/pe/libsci/24.07.0/INTEL/2023.2/x86_64/lib:/usr/lib64/gcc/x86_64-suse-linux/12:/opt/intel/oneapi/compiler/2024.2/lib' extra_rpaths: diff --git a/configs/sites/tier1/gaea-c6/packages.yaml b/configs/sites/tier1/gaea-c6/packages.yaml index 08d512e36..03364d66b 100644 --- a/configs/sites/tier1/gaea-c6/packages.yaml +++ b/configs/sites/tier1/gaea-c6/packages.yaml @@ -28,11 +28,6 @@ packages: externals: - spec: binutils@2.41-150100.7.46.1 prefix: /usr - cmake: - buildable: false - externals: - - spec: cmake@3.27.9 - modules: [cmake/3.27.9] coreutils: externals: - spec: coreutils@8.32 diff --git a/spack b/spack index e4724c3d7..b3fad4c1c 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit e4724c3d7952434f55f95b45a2276dec03c92949 +Subproject commit b3fad4c1c4803a83aa9cd2161003833ca721cf21 From 7d35bc0c42b1e100d3cc5bf71c2f125de9c33e5f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 3 Sep 2025 05:36:27 -0600 Subject: [PATCH 105/163] .github/workflows/ubuntu-ci-x86_64-gnu.yaml.tmpoff -> .github/workflows/ubuntu-ci-x86_64-gnu.yaml --- ...ubuntu-ci-x86_64-gnu.yaml.tmpoff => ubuntu-ci-x86_64-gnu.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu-ci-x86_64-gnu.yaml.tmpoff => ubuntu-ci-x86_64-gnu.yaml} (100%) diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml.tmpoff b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml similarity index 100% rename from .github/workflows/ubuntu-ci-x86_64-gnu.yaml.tmpoff rename to .github/workflows/ubuntu-ci-x86_64-gnu.yaml From 9ba4cfbe83b6c3f8338421905466dda474bf5f98 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 3 Sep 2025 08:49:25 -0600 Subject: [PATCH 106/163] .github/workflows/ubuntu-ci-x86_64-gnu.yaml: neptune-env -> neptune-python-env --- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index d84f923e5..8112cac55 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -145,10 +145,10 @@ jobs: echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.${ENVNAME}.base-env spack buildcache create -u local-binary base-env - # neptune-env + # neptune-python-env echo "neptune-env ..." - spack install --fail-fast --source --no-check-signature neptune-env 2>&1 | tee log.install.${ENVNAME}.neptune-env - spack buildcache create -u local-binary neptune-env + spack install --fail-fast --source --no-check-signature neptune-python-env 2>&1 | tee log.install.${ENVNAME}.neptune-env + spack buildcache create -u local-binary neptune-python-env fi # the rest From 2cb7124f05b0417ccc550d8d7087eda382514c29 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 4 Sep 2025 06:55:02 -0600 Subject: [PATCH 107/163] Update release_tasklist.md: fix invalid syntax (#1765) This PR fixes the syntax in `release_tasklist.md` following the instructions in https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-issue-forms. --- .github/ISSUE_TEMPLATE/release_tasklist.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release_tasklist.md b/.github/ISSUE_TEMPLATE/release_tasklist.md index 6ff9dc50a..4c1a8bf1e 100644 --- a/.github/ISSUE_TEMPLATE/release_tasklist.md +++ b/.github/ISSUE_TEMPLATE/release_tasklist.md @@ -5,19 +5,19 @@ title: '[RELEASE] release-x.y.z tasks' labels: '' assignees: # EPIC -- @natalie-perlin -- @ratkovasic-noaa -- @rickgrubin-noaa +- natalie-perlin +- ratkovasic-noaa +- rickgrubin-noaa # JCSDA -- @ashley314 -- @eap +- ashley314 +- eap # NASA -- @mathomp4 +- mathomp4 # NOAA -- @alexanderrichert-noaa -- @hanglei-noaa +- alexanderrichert-noaa +- hanglei-noaa # NRL -- @climbfuji +- climbfuji --- From b8e4add5a434b2c357f8104b1538d86bac132621 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 8 Sep 2025 11:13:31 -0600 Subject: [PATCH 108/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 2358b78d1..81eac568c 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 2358b78d1bc5cb1a2e862ed74347ee416e4ac9d0 +Subproject commit 81eac568c1a04199d2242c22c974686852d39282 From e5ab2d3e36bf14872e6dcde79a2dfa88e74e0fe7 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 8 Sep 2025 12:56:57 -0600 Subject: [PATCH 109/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 81eac568c..50857d8ea 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 81eac568c1a04199d2242c22c974686852d39282 +Subproject commit 50857d8ea172bb39aac4296feb446020b4249f7d From 5b859bbc293858678dfa2b49276b0965baafa4ec Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 9 Sep 2025 09:39:34 -0600 Subject: [PATCH 110/163] [skip ci] Update Blueback gcc try 1 --- ...ilers.yaml => compilers.yaml.nolongerused} | 0 .../tier1/blueback/packages_gcc-13.3.0.yaml | 34 +++++++++++++++++++ .../sites/tier1/blueback/packages_gcc.yaml | 17 ---------- 3 files changed, 34 insertions(+), 17 deletions(-) rename configs/sites/tier1/blueback/{compilers.yaml => compilers.yaml.nolongerused} (100%) create mode 100644 configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml delete mode 100644 configs/sites/tier1/blueback/packages_gcc.yaml diff --git a/configs/sites/tier1/blueback/compilers.yaml b/configs/sites/tier1/blueback/compilers.yaml.nolongerused similarity index 100% rename from configs/sites/tier1/blueback/compilers.yaml rename to configs/sites/tier1/blueback/compilers.yaml.nolongerused diff --git a/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml new file mode 100644 index 000000000..709202fdc --- /dev/null +++ b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml @@ -0,0 +1,34 @@ +packages: + mpi: + buildable: False + require: + - "cray-mpich@8.1.32" + gcc: + externals: + - spec: gcc@13.3.0 languages:='c,c++,fortran' + prefix: /usr + modules: + - PrgEnv-gnu/8.6.0 + - gcc-native/13.2 + - libfabric/1.22.0 + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + environment: + prepend_path: + LD_LIBRARY_PATH: '/opt/cray/libfabric/1.22.0/lib64' + set: + CONFIG_SITE: '' + CRAY_CPU_TARGET: x86-genoa + gcc-runtime: + externals: + - spec: gcc-runtime@13.3.0 + prefix: /usr + cray-mpich: + externals: + - spec: cray-mpich@8.1.32%gcc@13.3.0 ~wrappers + modules: + - cray-mpich/8.1.32 + - craype-network-ofi diff --git a/configs/sites/tier1/blueback/packages_gcc.yaml b/configs/sites/tier1/blueback/packages_gcc.yaml deleted file mode 100644 index be40b6e5a..000000000 --- a/configs/sites/tier1/blueback/packages_gcc.yaml +++ /dev/null @@ -1,17 +0,0 @@ -packages: - all: - compiler:: [gcc@13.3.0] - providers: - mpi:: [cray-mpich@8.1.32] - mpi: - buildable: False - cray-mpich: - externals: - - spec: cray-mpich@8.1.32%gcc@13.3.0 ~wrappers - modules: - - cray-mpich/8.1.32 - - craype-network-ofi - gcc-runtime: - externals: - - spec: gcc-runtime@13.3.0 - prefix: /usr From cf80ceb70ed2e808aed536c3e8e78b3f612a16cf Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 12 Sep 2025 05:21:46 -0600 Subject: [PATCH 111/163] [skip ci] Bug fix in configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml: ifx -> ifort --- configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml index 839d0144a..352d4b687 100644 --- a/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml +++ b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml @@ -16,7 +16,7 @@ packages: compilers: c: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/bin/icx cxx: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/bin/icpx - fortran: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/bin/ifx + fortran: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/bin/ifort environment: prepend_path: PATH: /cm/local/apps/gcc/11.2.0/bin From d45bf640c319bc49d9b4c4aefc99d2586ba6e510 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 12 Sep 2025 22:06:24 +0000 Subject: [PATCH 112/163] [skip ci] Update navy-aws site config --- configs/sites/tier1/navy-aws/bootstrap.yaml | 9 +++ configs/sites/tier1/navy-aws/config.yaml | 8 +-- .../tier1/navy-aws/packages_gcc-13.3.1.yaml | 32 +++++++++ .../navy-aws/packages_oneapi-2024.2.1.yaml | 16 ++--- .../navy-aws/packages_oneapi-2025.1.1.yaml | 65 ------------------- .../navy-aws/packages_oneapi-2025.2.0.yaml | 64 ++++++++++++++++++ 6 files changed, 117 insertions(+), 77 deletions(-) create mode 100644 configs/sites/tier1/navy-aws/bootstrap.yaml create mode 100644 configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml delete mode 100644 configs/sites/tier1/navy-aws/packages_oneapi-2025.1.1.yaml create mode 100644 configs/sites/tier1/navy-aws/packages_oneapi-2025.2.0.yaml diff --git a/configs/sites/tier1/navy-aws/bootstrap.yaml b/configs/sites/tier1/navy-aws/bootstrap.yaml new file mode 100644 index 000000000..8602ff6ff --- /dev/null +++ b/configs/sites/tier1/navy-aws/bootstrap.yaml @@ -0,0 +1,9 @@ +bootstrap: + sources: + - name: local-binaries + metadata: /project/spack-stack/spack-bootstrap-mirror/metadata/binaries + - name: local-sources + metadata: /project/spack-stack/spack-bootstrap-mirror/metadata/sources + trusted: + local-sources: true + local-binaries: true diff --git a/configs/sites/tier1/navy-aws/config.yaml b/configs/sites/tier1/navy-aws/config.yaml index b30215cb2..dd48c4196 100644 --- a/configs/sites/tier1/navy-aws/config.yaml +++ b/configs/sites/tier1/navy-aws/config.yaml @@ -4,7 +4,7 @@ config: # Overrides for spack build and staging areas to speed up builds # and avoid errors with hard links on the NFS filesystem /contrib - build_stage: /work/spack-stack-cache/build_stage - test_stage: /work/spack-stack-cache/test_stage - source_cache: /work/spack-stack-cache/source_cache - misc_cache: /work/spack-stack-cache/misc_cache + build_stage: ${HOME}/spack-stack-cache/build_stage + test_stage: ${HOME}/spack-stack-cache/test_stage + source_cache: ${HOME}/spack-stack-cache/source_cache + misc_cache: ${HOME}/spack-stack-cache/misc_cache diff --git a/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml b/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml new file mode 100644 index 000000000..497e94c7c --- /dev/null +++ b/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml @@ -0,0 +1,32 @@ +packages: + mpi: + buildable: False + require: + - "openmpi@4.1.1" + gcc: + externals: + - spec: gcc@13.3.1 languages:='c,c++,fortran' + prefix: /opt/rh/gcc-toolset-13/root + extra_attributes: + compilers: + c: /opt/rh/gcc-toolset-13/root/bin/gcc + cxx: /opt/rh/gcc-toolset-13/root/bin/g++ + fortran: /opt/rh/gcc-toolset-13/root/bin/gfortran + #- spec: gcc@11.5.0 languages:='c,c++,fortran' + # prefix: /usr + # extra_attributes: + # compilers: + # c: /usr/bin/gcc + # cxx: /usr/bin/g++ + # fortran: /usr/bin/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@13.3.1%gcc@13.3.1 + prefix: /opt/rh/gcc-toolset-13/root + openmpi: + externals: + - spec: openmpi@4.1.1 + prefix: /usr/lib64/openmpi + modules: + - mpi/openmpi-x86_64 + diff --git a/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml index ea9f04880..8ef29244e 100644 --- a/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml +++ b/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml @@ -6,35 +6,35 @@ packages: intel-oneapi-compilers: externals: - spec: intel-oneapi-compilers@2024.2.1 - prefix: /contrib/oneapi/2024.2.1 + prefix: /project/oneapi/2024.2.1 modules: - tbb/2021.13 - compiler-rt/2024.2.1 - compiler/2024.2.1 extra_attributes: compilers: - c: /contrib/oneapi/2024.2.1/compiler/2024.2/bin/icx - cxx: /contrib/oneapi/2024.2.1/compiler/2024.2/bin/icpx - fortran: /contrib/oneapi/2024.2.1/compiler/2024.2/bin/ifort + c: /project/oneapi/2024.2.1/compiler/2024.2/bin/icx + cxx: /project/oneapi/2024.2.1/compiler/2024.2/bin/icpx + fortran: /project/oneapi/2024.2.1/compiler/2024.2/bin/ifort intel-oneapi-mpi: buildable: false externals: - spec: intel-oneapi-mpi@2021.13 - prefix: /contrib/oneapi/2024.2.1 + prefix: /project/oneapi/2024.2.1 modules: - mpi/2021.13 intel-oneapi-mkl: buildable: false externals: - spec: intel-oneapi-mkl@2024.2 - prefix: /contrib/oneapi/2024.2.1 + prefix: /project/oneapi/2024.2.1 modules: - mkl/2024.2 intel-oneapi-tbb: buildable: false externals: - spec: intel-oneapi-tbb@2021.13 - prefix: /contrib/oneapi/2024.2.1 + prefix: /project/oneapi/2024.2.1 modules: - tbb/2021.13 intel-oneapi-runtime: @@ -42,7 +42,7 @@ packages: #buildable: false externals: - spec: intel-oneapi-runtime@2024.2.1 - prefix: /contrib/oneapi/2024.2.1 + prefix: /project/oneapi/2024.2.1 modules: - tbb/2021.13 - compiler-rt/2024.2.1 diff --git a/configs/sites/tier1/navy-aws/packages_oneapi-2025.1.1.yaml b/configs/sites/tier1/navy-aws/packages_oneapi-2025.1.1.yaml deleted file mode 100644 index af81fd950..000000000 --- a/configs/sites/tier1/navy-aws/packages_oneapi-2025.1.1.yaml +++ /dev/null @@ -1,65 +0,0 @@ -packages: - mpi: - buildable: False - require: - - "intel-oneapi-mpi@2021.15" - intel-oneapi-compilers: - externals: - - spec: intel-oneapi-compilers@2025.1.1 - prefix: /contrib/oneapi/2025.1.1 - modules: - - umf/0.10.0 - - tbb/2022.1 - - compiler-rt/2025.1.1 - - compiler/2025.1.1 - extra_attributes: - compilers: - c: /contrib/oneapi/2025.1.1/compiler/2025.1/bin/icx - cxx: /contrib/oneapi/2025.1.1/compiler/2025.1/bin/icpx - fortran: /contrib/oneapi/2025.1.1/compiler/2025.1/bin/ifx - - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.15 - prefix: /contrib/oneapi/2025.1.1 - modules: - - mpi/2021.15 - intel-oneapi-mkl: - buildable: false - externals: - - spec: intel-oneapi-mkl@2025.1 - prefix: /contrib/oneapi/2025.1.1 - modules: - - mkl/2025.1 - intel-oneapi-tbb: - buildable: false - externals: - - spec: intel-oneapi-tbb@2022.2 - prefix: /contrib/oneapi/2025.1.1 - modules: - - tbb/2022.1 - intel-oneapi-runtime: - # LINK TO BUG REPORT HERE - #buildable: false - externals: - - spec: intel-oneapi-runtime@2025.1.1 - prefix: /contrib/oneapi/2025.1.1 - modules: - - umf/0.10.0 - - tbb/2022.1 - - compiler-rt/2025.1.1 - - compiler/2025.1.1 - gcc: - externals: - - spec: gcc@11.5.0 languages:='c,c++' - prefix: /usr - extra_attributes: - compilers: - c: /usr/bin/gcc - cxx: /usr/bin/g++ - fortran: /usr/bin/gfortran - gcc-runtime: - externals: - - spec: gcc-runtime@11.5.0 - prefix: /usr diff --git a/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.0.yaml b/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.0.yaml new file mode 100644 index 000000000..e76a98e50 --- /dev/null +++ b/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.0.yaml @@ -0,0 +1,64 @@ +packages: + mpi: + buildable: False + require: + - "intel-oneapi-mpi@2021.16" + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2025.2.0 + prefix: /project/oneapi/2025.2.0 + modules: + - umf/0.11.0 + - tbb/2022.2 + - compiler-rt/2025.2.0 + - compiler/2025.2.0 + extra_attributes: + compilers: + c: /project/oneapi/2025.2.0/compiler/2025.2/bin/icx + cxx: /project/oneapi/2025.2.0/compiler/2025.2/bin/icpx + fortran: /project/oneapi/2025.2.0/compiler/2025.2/bin/ifx + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.16 + prefix: /project/oneapi/2025.2.0 + modules: + - mpi/2021.16 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.2 + prefix: /project/oneapi/2025.2.0 + modules: + - mkl/2025.2 + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2022.2 + prefix: /project/oneapi/2025.2.0 + modules: + - tbb/2022.2 + intel-oneapi-runtime: + # LINK TO BUG REPORT HERE + #buildable: false + externals: + - spec: intel-oneapi-runtime@2025.2.0 + prefix: /project/oneapi/2025.2.0 + modules: + - umf/0.11.0 + - tbb/2022.2 + - compiler-rt/2025.2.0 + - compiler/2025.2.0 + gcc: + externals: + - spec: gcc@11.5.0 languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@11.5.0 + prefix: /usr From a18dab1ac99e9cacf27fdb195467081514be6150 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 12 Sep 2025 16:11:59 -0600 Subject: [PATCH 113/163] [skip ci] more updates for navy-aws site, and new compiler bounty oneapi-2025.2.1 --- configs/sites/tier1/navy-aws/modules.yaml | 1 + .../tier1/navy-aws/packages_gcc-13.3.1.yaml | 8 ++-- .../bounty/packages_oneapi-2025.2.1.yaml | 43 +++++++++++++++++++ 3 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 configs/sites/tier2/bounty/packages_oneapi-2025.2.1.yaml diff --git a/configs/sites/tier1/navy-aws/modules.yaml b/configs/sites/tier1/navy-aws/modules.yaml index b134d3669..293b62d4e 100644 --- a/configs/sites/tier1/navy-aws/modules.yaml +++ b/configs/sites/tier1/navy-aws/modules.yaml @@ -5,3 +5,4 @@ modules: tcl: include: - python + - openmpi diff --git a/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml b/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml index 497e94c7c..1a584cff1 100644 --- a/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml +++ b/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml @@ -2,7 +2,7 @@ packages: mpi: buildable: False require: - - "openmpi@4.1.1" + - "openmpi@4.1.8" gcc: externals: - spec: gcc@13.3.1 languages:='c,c++,fortran' @@ -25,8 +25,6 @@ packages: prefix: /opt/rh/gcc-toolset-13/root openmpi: externals: - - spec: openmpi@4.1.1 - prefix: /usr/lib64/openmpi - modules: - - mpi/openmpi-x86_64 + - spec: openmpi@4.1.8 + prefix: /project/gcc/openmpi-4.1.8/gcc-13.3.1 diff --git a/configs/sites/tier2/bounty/packages_oneapi-2025.2.1.yaml b/configs/sites/tier2/bounty/packages_oneapi-2025.2.1.yaml new file mode 100644 index 000000000..e24d32a6c --- /dev/null +++ b/configs/sites/tier2/bounty/packages_oneapi-2025.2.1.yaml @@ -0,0 +1,43 @@ +packages: + mpi: + buildable: False + require: + - "intel-oneapi-mpi@2021.16" + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2025.2.1 + prefix: /opt/intel/oneapi + extra_attributes: + compilers: + c: /opt/intel/oneapi/2025.2/bin/icx + cxx: /opt/intel/oneapi/2025.2/bin/icpx + fortran: /opt/intel/oneapi/2025.2/bin/ifx + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.16 + prefix: /opt/intel/oneapi + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.2 + prefix: /opt/intel/oneapi + intel-oneapi-runtime: + # LINK TO BUG REPORT HERE + #buildable: false + externals: + - spec: intel-oneapi-runtime@2025.2.1 %intel-oneapi-compilers@2025.2.1 + prefix: /opt/intel/oneapi + gcc: + externals: + - spec: gcc@11.5.0 languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + gcc-runtime: + externals: + - spec: gcc-runtime@11.5.0 %gcc@11.5.0 + prefix: /usr From b39a46bdeb88beeb17b60bad3fa2c582b9a6b5a0 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 15 Sep 2025 08:03:41 -0600 Subject: [PATCH 114/163] spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py: bug fix to set compiler flags correctly --- spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 3272441f8..1a67ae1d9 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -440,7 +440,7 @@ def custom_sort_key(entry): # Compiler flags; names are lowercase in spack if "flags" in compiler.extra_attributes.keys(): for flag_name in compiler.extra_attributes["flags"].keys(): - flag_values = compiler["compiler"]["flags"][flag_name] + flag_values = compiler.extra_attributes["flags"][flag_name] substitutes["COMPFLAGS"] += setenv_command( module_choice, flag_name.upper(), flag_values ) From 6dda147f1d71c6510fba7e0b14c73405aa067a1c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 15 Sep 2025 11:47:02 -0600 Subject: [PATCH 115/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 50857d8ea..a49c3a65c 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 50857d8ea172bb39aac4296feb446020b4249f7d +Subproject commit a49c3a65c8e0ec91239e4c2849c09b68e9d384a3 From a5e8259d2f00709971de2343ca55ced8aa08836d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 23 Sep 2025 10:39:18 -0600 Subject: [PATCH 116/163] Update configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml --- configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml index 709202fdc..6c02eaef2 100644 --- a/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml +++ b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml @@ -4,6 +4,7 @@ packages: require: - "cray-mpich@8.1.32" gcc: + buildable: False externals: - spec: gcc@13.3.0 languages:='c,c++,fortran' prefix: /usr @@ -23,6 +24,7 @@ packages: CONFIG_SITE: '' CRAY_CPU_TARGET: x86-genoa gcc-runtime: + buildable: False externals: - spec: gcc-runtime@13.3.0 prefix: /usr From c68c61c5f7f5161b3bd829ed3e641dda268a0d29 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 25 Sep 2025 09:38:30 -0600 Subject: [PATCH 117/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index a49c3a65c..d974db373 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit a49c3a65c8e0ec91239e4c2849c09b68e9d384a3 +Subproject commit d974db3739f2c83ace0f1459dbc3956feabc9e88 From 7159b6668873de8bc5c7b9a9e61905b6f79fb742 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 23 Sep 2025 10:39:18 -0600 Subject: [PATCH 118/163] Update configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml --- configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml | 2 ++ spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml index 709202fdc..6c02eaef2 100644 --- a/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml +++ b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml @@ -4,6 +4,7 @@ packages: require: - "cray-mpich@8.1.32" gcc: + buildable: False externals: - spec: gcc@13.3.0 languages:='c,c++,fortran' prefix: /usr @@ -23,6 +24,7 @@ packages: CONFIG_SITE: '' CRAY_CPU_TARGET: x86-genoa gcc-runtime: + buildable: False externals: - spec: gcc-runtime@13.3.0 prefix: /usr diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 1a67ae1d9..548f8bddf 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -554,6 +554,11 @@ def custom_sort_key(entry): substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) + # Set mpi_ROOT and MPI_ROOT (the latter seems to be needed for Cray) + substitutes["MPIROOT"] = setenv_command(module_choice, "mpi_ROOT", mpi_provider.prefix) + substitutes["MPIROOT"] += "\n" + setenv_command(module_choice, "MPI_ROOT", mpi_provider.prefix) + logging.debug(" ... ... MPIROOT: {}".format(substitutes["MPIROOT"])) + # Compiler wrapper environment variables if mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers": substitutes["MPICC"] = os.path.join("mpiicx") From 220d786a1681f3a1806a614b4b3f93c63f560e7e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 29 Sep 2025 08:29:55 -0600 Subject: [PATCH 119/163] [skip ci] Try better setting MPI environment variables in meta modules --- .../tier2/bounty/packages_clang-20.1.5.yaml | 16 - .../tier2/bounty/packages_clang-21.1.1.yaml | 16 + .../spack-stack/stack/meta_modules.py | 34 +- .../stack/meta_modules.py.backup.20250730 | 707 ------------------ .../stack/meta_modules.py.backup.20250801 | 687 ----------------- .../jcsda-emc/spack-stack/stack/templates/mpi | 16 +- .../spack-stack/stack/templates/mpi.lua | 12 +- 7 files changed, 56 insertions(+), 1432 deletions(-) delete mode 100644 configs/sites/tier2/bounty/packages_clang-20.1.5.yaml create mode 100644 configs/sites/tier2/bounty/packages_clang-21.1.1.yaml delete mode 100755 spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250730 delete mode 100755 spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250801 diff --git a/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml b/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml deleted file mode 100644 index 4f2724b58..000000000 --- a/configs/sites/tier2/bounty/packages_clang-20.1.5.yaml +++ /dev/null @@ -1,16 +0,0 @@ -packages: - mpi: - require: - - "openmpi@=5.0.8" - llvm: - externals: - - spec: llvm@20.1.5 +flang ~libomptarget - prefix: /home/dom/prod/llvm-20.1.5 - extra_attributes: - compilers: - c: /home/dom/prod/llvm-20.1.5/bin/clang - cxx: /home/dom/prod/llvm-20.1.5/bin/clang++ - fortran: /home/dom/prod/llvm-20.1.5/bin/flang-new - extra_rpaths: - - /home/dom/prod/llvm-20.1.5/lib - - /home/dom/prod/llvm-20.1.5/lib/x86_64-unknown-linux-gnu diff --git a/configs/sites/tier2/bounty/packages_clang-21.1.1.yaml b/configs/sites/tier2/bounty/packages_clang-21.1.1.yaml new file mode 100644 index 000000000..fa181f9fe --- /dev/null +++ b/configs/sites/tier2/bounty/packages_clang-21.1.1.yaml @@ -0,0 +1,16 @@ +packages: + mpi: + require: + - "openmpi@=5.0.8" + llvm: + externals: + - spec: llvm@21.1.1 +flang ~libomptarget + prefix: /home/dom/prod/llvm-21.1.1 + extra_attributes: + compilers: + c: /home/dom/prod/llvm-21.1.1/bin/clang + cxx: /home/dom/prod/llvm-21.1.1/bin/clang++ + fortran: /home/dom/prod/llvm-21.1.1/bin/flang-new + extra_rpaths: + - /home/dom/prod/llvm-21.1.1/lib + - /home/dom/prod/llvm-21.1.1/lib/x86_64-unknown-linux-gnu diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 548f8bddf..90665543d 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -554,33 +554,35 @@ def custom_sort_key(entry): substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) - # Set mpi_ROOT and MPI_ROOT (the latter seems to be needed for Cray) + # Set mpi_ROOT, MPI_ROOT, and other versions of it used by Cray substitutes["MPIROOT"] = setenv_command(module_choice, "mpi_ROOT", mpi_provider.prefix) - substitutes["MPIROOT"] += "\n" + setenv_command(module_choice, "MPI_ROOT", mpi_provider.prefix) + substitutes["MPIROOT"] += setenv_command(module_choice, "MPI_ROOT", mpi_provider.prefix) + substitutes["MPIROOT"] += setenv_command(module_choice, "MPI_HOME", mpi_provider.prefix) + substitutes["MPIROOT"] += setenv_command(module_choice, "MPICH_DIR", mpi_provider.prefix) logging.debug(" ... ... MPIROOT: {}".format(substitutes["MPIROOT"])) # Compiler wrapper environment variables if mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers": - substitutes["MPICC"] = os.path.join("mpiicx") - substitutes["MPICXX"] = os.path.join("mpiicpx") + substitutes["MPICC"] = os.path.join(mpi_provider.prefix, "mpiicx") + substitutes["MPICXX"] = os.path.join(mpi_provider.prefix, "mpiicpx") if "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and not "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: - substitutes["MPIF77"] = os.path.join("mpiifx") - substitutes["MPIF90"] = os.path.join("mpiifx") + substitutes["MPIF77"] = os.path.join(mpi_provider.prefix, "mpiifx") + substitutes["MPIF90"] = os.path.join(mpi_provider.prefix, "mpiifx") elif not "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: - substitutes["MPIF77"] = os.path.join("mpiifort") - substitutes["MPIF90"] = os.path.join("mpiifort") + substitutes["MPIF77"] = os.path.join(mpi_provider.prefix, "mpiifort") + substitutes["MPIF90"] = os.path.join(mpi_provider.prefix, "mpiifort") else: raise Exception(f"For {mpi_provider.name}, cannot determine MPI wrapper from FC={COMPILER_SUBSTITUTES_SAVE['FC']}") elif mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers-classic": - substitutes["MPICC"] = os.path.join("mpiicc") - substitutes["MPICXX"] = os.path.join("mpiicpc") - substitutes["MPIF77"] = os.path.join("mpiifort") - substitutes["MPIF90"] = os.path.join("mpiifort") + substitutes["MPICC"] = os.path.join(mpi_provider.prefix, "mpiicc") + substitutes["MPICXX"] = os.path.join(mpi_provider.prefix, "mpiicpc") + substitutes["MPIF77"] = os.path.join(mpi_provider.prefix, "mpiifort") + substitutes["MPIF90"] = os.path.join(mpi_provider.prefix, "mpiifort") else: - substitutes["MPICC"] = os.path.join("mpicc") - substitutes["MPICXX"] = os.path.join("mpic++") - substitutes["MPIF77"] = os.path.join("mpif77") - substitutes["MPIF90"] = os.path.join("mpif90") + substitutes["MPICC"] = os.path.join(mpi_provider.prefix, "mpicc") + substitutes["MPICXX"] = os.path.join(mpi_provider.prefix, "mpic++") + substitutes["MPIF77"] = os.path.join(mpi_provider.prefix, "mpif77") + substitutes["MPIF90"] = os.path.join(mpi_provider.prefix, "mpif90") # Also set the direct compiler environment variables substitutes["CC"] = COMPILER_SUBSTITUTES_SAVE["CC"] substitutes["CXX"] = COMPILER_SUBSTITUTES_SAVE["CXX"] diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250730 b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250730 deleted file mode 100755 index 25dd2446f..000000000 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250730 +++ /dev/null @@ -1,707 +0,0 @@ -#!/usr/bin/env python3 - -import copy -import logging -import os -import re -import sys - -import spack -import spack.environment as ev -# import spack.repo -from spack.provider_index import ProviderIndex - -# logging.basicConfig(level=logging.INFO) -logging.basicConfig(format="%(message)s", level=logging.DEBUG) - -# Get basic directory information -logging.info("Configuring basic directory information ...") -this_script_dir = os.path.realpath(os.path.split(__file__)[0]) -base_dir = os.path.realpath(os.path.join(this_script_dir, "..")) -spack_dir = spack.paths.spack_root -logging.info(" ... script directory: {}".format(this_script_dir)) -logging.info(" ... base directory: {}".format(base_dir)) -logging.info(" ... spack directory: {}".format(spack_dir)) - -# Templates for creating compiler modules -COMPILER_TEMPLATES = { - "lmod": os.path.join(this_script_dir, "templates/compiler.lua"), - "tcl": os.path.join(this_script_dir, "templates/compiler"), -} -MPI_TEMPLATES = { - "lmod": os.path.join(this_script_dir, "templates/mpi.lua"), - "tcl": os.path.join(this_script_dir, "templates/mpi"), -} -MODULE_FILE_EXTENSION = { - "lmod": ".lua", - "tcl": "" -} - -SUBSTITUTES_TEMPLATE = { - "MODULELOADS": "", - "MODULEPREREQS": "", - "MODULEPATHS": "", - "CC": "", - "CXX": "", - "F77": "", - "FC": "", - "COMPFLAGS": "", - "ENVVARS": "", - "MPICC": "", - "MPICXX": "", - "MPIF77": "", - "MPIROOT": "", -} - - -COMPILER_TRANSLATION_TABLE = { - "gcc" : "gcc", - "clang" : "llvm", - "intel" : "intel-oneapi-compilers-classic", - "oneapi" : "intel-oneapi-compilers", -} - - -def setenv_command(module_choice, key, value): - if module_choice == "lmod": - return 'setenv("{}", "{}")\n'.format(key, value) - else: - return "setenv {{{}}} {{{}}}\n".format(key, value) - - -def unsetenv_command(module_choice, key, value): - if module_choice == "lmod": - return 'unsetenv("{}", "{}")\n'.format(key, value) - else: - return "unsetenv {{{}}} {{{}}}\n".format(key, value) - - -def append_path_command(module_choice, key, value): - if module_choice == "lmod": - return 'append_path("{}", "{}")\n'.format(key, value) - else: - return "append-path {{{}}} {{{}}}\n".format(key, value) - - -def prepend_path_command(module_choice, key, value): - if module_choice == "lmod": - return 'prepend_path("{}", "{}")\n'.format(key, value) - else: - return "prepend-path {{{}}} {{{}}}\n".format(key, value) - - -def module_load_command(module_choice, module): - if module_choice == "lmod": - return 'load("{}")\n'.format(module) - else: - return """if {{ [ module-info mode load ] && ![ is-loaded {0} ] }} {{ - module load {0} -}}\n""".format( - module - ) - - -def module_prereq_command(module_choice, module): - if module_choice == "lmod": - return 'prereq("{}")\n'.format(module) - else: - return "" - - -def modulepath_prepend_command(module_choice, modulepath): - if module_choice == "lmod": - return 'prepend_path("MODULEPATH", "{}")\n'.format(modulepath) - else: - return "prepend-path {{MODULEPATH}} {{{}}}\n".format(modulepath) - - -def substitute_config_vars(config_str): - """ - Substitute spack-specific and environment variables that may be present - in configuration files. See: - https://spack.readthedocs.io/en/latest/configuration.html#config-file-variables - """ - spack_vars = { - "ENV": ev.active_environment().path, - "SPACK": os.getenv("SPACK_ROOT"), - "TEMPDIR": None, - "USER": os.getenv("HOME"), - "USER_CACHE_PATH": os.path.join(os.getenv("HOME"), ".spack"), - } - - if config_str.startswith("~"): - config_str = config_str.replace("~", os.getenv("HOME")) - - # Get var as it appears in the string (e.g. ${env}), and its name (e.g. env) - matches = re.findall(r"(\$(\w+))|(\${(\w+)})", config_str) - for match in matches: - if match[0]: - pair = (match[0], match[1]) - else: - pair = (match[2], match[3]) - - var_string = pair[0] - var_name = pair[1].upper() - - sub_value = spack_vars[var_name] if spack_vars[var_name] else os.getenv(var_name) - config_str = config_str.replace(var_string, sub_value) - - return config_str - - -def setup_meta_modules(): - # Find currently active spack environment, activate here - logging.info("Configuring active spack environment ...") - env_dir = ev.active_environment().path - if not env_dir: - raise Exception("No active spack environment") - env = spack.environment.Environment(env_dir) - spack.environment.environment.activate(env) - logging.info(" ... environment directory: {}".format(env_dir)) - - # Parse spack main config from environment - logging.info("Parsing spack environment main config ...") - main_config = spack.config.get("config") - install_dir = substitute_config_vars(main_config["install_tree"]["root"]) - - if not os.path.isabs(install_dir): - install_dir = os.path.realpath(os.path.join(env_dir, install_dir)) - else: - install_dir = os.path.realpath(install_dir) - logging.info(" ... install directory: {}".format(install_dir)) - - # Parse spack module config from environment - logging.info("Parsing spack environment modules config ...") - module_config = spack.config.get("modules") - - # Check which modules are used - tcl or lmod (can only be one) - if len(module_config["default"]["enable"]) > 1: - raise Exception("Can use either lmod or tcl modules, not both") - module_choice = module_config["default"]["enable"][0] - logging.info(" ... configured to use {} modules".format(module_choice)) - - # Need to set a few variables when tcl modules are used - if module_choice == "tcl": - module_replace_patterns = ["is-loaded", "module load", "depends-on"] - # sed syntax differs on macOS - if sys.platform == "darwin": - sed_syntax_fix = "''" - else: - sed_syntax_fix = "" - - # Top-level module directory - module_dir = substitute_config_vars(module_config["default"]["roots"][module_choice]) - if not os.path.isabs(module_dir): - module_dir = os.path.realpath(os.path.join(env_dir, module_dir)) - else: - module_dir = os.path.realpath(module_dir) - logging.info(f" ... module directory: {module_dir}") - - # Get all specs and determine compilers and mpi providers - hashes = env.all_hashes() - specs = spack.store.STORE.db.query(hashes=hashes) - q = ProviderIndex(specs=specs, repository=spack.repo.PATH) - - c_providers = q.providers_for("c") - cxx_providers = q.providers_for("cxx") - fortran_providers = q.providers_for("fortran") - compilers = list(set(c_providers + cxx_providers + fortran_providers)) - if not compilers: - raise Exception("No compilers found") - logging.info(f" ... compilers: {compilers}") - - # To remove compiler prefices from tcl modulefiles, we need - # a mock compiler "none@none" for external packages - if module_choice == "tcl": - mock_compiler_list = [x.name+"@"+str(x.version) for x in compilers] + ["none@none"] - logging.debug(f" ... mock_compiler_list: {mock_compiler_list}") - - mpi_providers = q.providers_for("mpi") - if not len(mpi_providers)==1: - raise Exception(f"Expected exactly one MPI provider, but got {mpi_providers}") - logging.info(f" ... mpi_providers: {mpi_providers}") - - # Core compilers is only a valid options for lmod - if module_choice == "lmod": - # Determine core compiler(s) and make sure they are not used - core_compilers = module_config["default"][module_choice]["core_compilers"] - logging.info(" ... core compilers: {}".format(core_compilers)) - # Check that none of the compilers used for the stack is a core compiler - for core_compiler in core_compilers: - if any(core_compiler in x for x in flattened_compiler_list): - raise Exception( - """Not supported: compiler used for environment - is in list of core compilers""" - ) - - # Prepare meta module directory - logging.info("Preparing meta module directory ...") - meta_module_dir = os.path.join(module_dir, "Core") - if not os.path.isdir(meta_module_dir): - os.mkdir(meta_module_dir) - logging.info(" ... meta module directory : {}".format(meta_module_dir)) - - - # DH* 20250730 - DO WE NEED BOTH WAYS TO DETERMINE THE PREFERRED COMPILER? - # MAYBE KEEP FOR NOW UNTIL WE KNOW BETTER WHAT IS NEEDED AND WHAT NOT. - - # Determine the preferred compiler and sort the list of compilers so that - # the preferred compiler comes last. This is so that all other compilers - # populate the MODULEPATHS_SAVE list before the preferred compiler - # takes it and adds it to the stack-COMPILER metamodule. Likewise, we need - # to save the list of compiler substitutions from the preferred compiler - # so that we have access to it when we build the MPI meta module. - try: - preferred_compilers = spack.config.get("packages")["all"]["require"] - except: - raise Exception( - """Unable to detect preferred compiler from environment. - Does the environment have the config entry 'packages:all:require?'""" - ) - if len(preferred_compilers)>1: - raise Exception(f"Invalid value for packages:all:require is {preferred_compilers}") - match = re.search(r'%\[when=%fortran\](\S+)', preferred_compilers[0]) - if match and match.group(1) in COMPILER_TRANSLATION_TABLE.keys(): - preferred_compiler_v1 = COMPILER_TRANSLATION_TABLE[match.group(1)] - else: - raise Exception(f"Invalid value for packages:all:require {preferred_compilers[0]}") - - try: - preferred_compilers = spack.config.get("packages")["fortran"]["require"] - except: - raise Exception( - """Unable to detect preferred compiler from environment. - Does the environment have the config entry 'packages:fortran:require?'""" - ) - if len(preferred_compilers)>1: - raise Exception(f"Invalid value for packages:fortran:require is {preferred_compilers}") - if preferred_compilers[0] in COMPILER_TRANSLATION_TABLE.values(): - preferred_compiler_v2 = preferred_compilers[0] - else: - raise Exception(f"Invalid value for packages:fortran:require {preferred_compilers[0]}") - if preferred_compiler_v1 == preferred_compiler_v2: - preferred_compiler = preferred_compiler_v1 - del preferred_compilers - del preferred_compiler_v1 - del preferred_compiler_v2 - else: - raise Exception(f"Multiple preferred compilers in spack config: {preferred_compiler_v1} vs {preferred_compiler_v2}") - logging.info(" ... preferred compiler: {}".format(preferred_compiler)) - # *DH 20250730 - - # Sort the list using a custom key - def custom_sort_key(entry): - # Return a tuple where the first element is 1 if the entry contains the word, else 0 - # The second element is the entry itself for natural sorting within groups - return (1 if preferred_compiler in entry else 0, entry) - compilers = sorted(compilers, key=custom_sort_key) - - # Create compiler modules - logging.info("Creating compiler modules ...") - - # Collect and save modulepaths for the preferred compiler - MODULEPATHS_SAVE = [] - - # Append modulepath for external specs - modulepath_save = os.path.join(module_dir, "none", "none") - if not os.path.isdir(modulepath_save): - os.makedirs(modulepath_save) - logging.info(" ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) - MODULEPATHS_SAVE.append(modulepath_save) - - # Initialize saved substitutes to None (populate for preferred compiler later) - COMPILER_SUBSTITUTES_SAVE = None - - for compiler in compilers: - logging.info(f" ... configuring compiler {compiler.name}@{compiler.version}") - - modulepath_save = os.path.join(module_dir, compiler.name, str(compiler.version)) - if not os.path.isdir(modulepath_save): - os.makedirs(modulepath_save) - logging.info(" ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) - MODULEPATHS_SAVE.append(modulepath_save) - - # For tcl modules remove the compiler prefices from the module contents - if module_choice == "tcl": - logging.info( - " ... ... removing compiler prefices from tcl modulefiles in {}".format( - modulepath_save - ) - ) - - for root, ddir, files in os.walk(modulepath_save): - for ffile in files: - filepath = os.path.join(root, ffile) - logging.debug( - " ... ... ... removing compiler prefices in {}".format(filepath) - ) - for pattern in module_replace_patterns: - for tmp_compiler in mock_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") - cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( - pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) - - # The remainder of the loop is only needed for the preferred compiler - if not compiler.name in preferred_compiler: - continue - - logging.info( - " ... configuring stack compiler {}@{}".format(compiler.name, compiler.version) - ) - compiler_module_dir = os.path.join(meta_module_dir, "stack-" + compiler.name) - compiler_module_file = os.path.join( - compiler_module_dir, str(compiler.version) + MODULE_FILE_EXTENSION[module_choice] - ) - substitutes = SUBSTITUTES_TEMPLATE.copy() - - if not compiler.external: - raise Exception(f"spack-built compiler {compiler} not yet supported") - - #### Check if the external package has a prefix - ###if compiler.prefix: - ### prefix = compiler.prefix - ### # Workaround for intel-oneapi-mpi. Because of the way this package - ### # is configured in spack, the prefix may be incomplete - ### if compiler.name == "intel-oneapi-compilers" or "intel-oneapi-compilers-classic": - ### if not os.path.isdir(os.path.join(prefix, "bin")): - ### prefix = os.path.join(prefix, "compiler", str(compiler.version)) - ### if not os.path.isdir(os.path.join(prefix, "bin")): - ### raise Exception("Invalid prefix for external compiler: {prefix}") - ###else: - ### prefix = None - - # Compiler environment variables; names are lowercase in spack - substitutes["CC"] = compiler.extra_attributes["compilers"]["c"] - substitutes["CXX"] = compiler.extra_attributes["compilers"]["cxx"] - substitutes["F77"] = compiler.extra_attributes["compilers"]["fortran"] - substitutes["FC"] = compiler.extra_attributes["compilers"]["fortran"] - logging.debug(" ... ... CC : {}".format(substitutes["CC"])) - logging.debug(" ... ... CXX : {}".format(substitutes["CXX"])) - logging.debug(" ... ... F77 : {}".format(substitutes["F77"])) - logging.debug(" ... ... FC' : {}".format(substitutes["FC"])) - - # Compiler flags; names are lowercase in spack - if "flags" in compiler.extra_attributes.keys(): - for flag_name in compiler.extra_attributes["flags"].keys(): - flag_values = compiler["compiler"]["flags"][flag_name] - substitutes["COMPFLAGS"] += setenv_command( - module_choice, flag_name.upper(), flag_values - ) - substitutes["COMPFLAGS"] = substitutes["COMPFLAGS"].rstrip("\n") - logging.debug(" ... ... COMPFLAGS: {}".format(substitutes["COMPFLAGS"])) - - if compiler.external_modules: - raise Exception(f"UNTESTED XYZ MODULES: {compiler.external_modules}") - # Existing non-spack modules to load - for module in compiler.external_modules: - substitutes["MODULELOADS"] += module_load_command(module_choice, module) - substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) - substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") - substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") - logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) - logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) - - # Environment variables; case-sensitive in spack - if "environment" in compiler.extra_attributes.keys(): - # DH* CAN WE SIMPLIFY THIS WITH A LOOP AND A GENERIC FUNCTION? - # append_path - if "append_path" in compiler.extra_attributes["environment"].keys(): - for env_name in compiler.extra_attributes["environment"]["append_path"]: - env_values = compiler.extra_attributes["environment"]["append_path"][env_name] - substitutes["ENVVARS"] += append_path_command( - module_choice, env_name, env_values - ) - # prepend_path - if "prepend_path" in compiler.extra_attributes["environment"].keys(): - for env_name in compiler.extra_attributes["environment"]["prepend_path"]: - env_values = compiler.extra_attributes["environment"]["prepend_path"][env_name] - substitutes["ENVVARS"] += prepend_path_command( - module_choice, env_name, env_values - ) - # set - if "set" in compiler.extra_attributes["environment"].keys(): - for env_name in compiler.extra_attributes["environment"]["set"]: - env_values = compiler.extra_attributes["environment"]["set"][env_name] - substitutes["ENVVARS"] += setenv_command( - module_choice, env_name, env_values - ) - # unset - if "unset" in compiler.extra_attributes["environment"].keys(): - for env_name in compiler.extra_attributes["environment"]["unset"]: - env_values = compiler.extra_attributes["environment"]["unset"][env_name] - substitutes["ENVVARS"] += unsetenv_command( - module_choice, env_name, env_values - ) - # *DH - substitutes["ENVVARS"] = substitutes["ENVVARS"].rstrip("\n") - logging.debug(" ... ... ENVVARS : {}".format(substitutes["ENVVARS"])) - - # Spack compiler module hierarchy - append all saved modulepaths - for modulepath in MODULEPATHS_SAVE: - substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) - substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") - logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) - - # Read compiler template into module_content string - with open(COMPILER_TEMPLATES[module_choice]) as f: - module_content = f.read() - - # Substitute variables in module_content - for key in substitutes.keys(): - module_content = module_content.replace("@{}@".format(key), substitutes[key]) - - # Write compiler lua module - if not os.path.isdir(compiler_module_dir): - os.makedirs(compiler_module_dir) - with open(compiler_module_file, "w") as f: - f.write(module_content) - logging.info(" ... writing {}".format(compiler_module_file)) - # If this is the last compiler in the list (i.e. the preferred compiler), - # then save the substitutes for later use for building the MPI meta module. - if compiler == compilers[-1]: - COMPILER_SUBSTITUTES_SAVE = substitutes - - del MODULEPATHS_SAVE - - # Collect and save modulepaths for MPI with preferred compiler - MODULEPATHS_SAVE = [] - - # Create mpi modules - as of July 2025, only one - # MPI provider is allowed (for the preferred compiler) - for mpi_provider in mpi_providers: - - for compiler in compilers: - logging.info( - " ... configuring stack mpi library {}@{} for compiler {}@{}".format( - mpi_provider.name, mpi_provider.version, compiler.name, compiler.version - ) - ) - - # Spack mpi+compiler module hierarchy - modulepath_save = os.path.join( - module_dir, mpi_provider.name, str(mpi_provider.version), compiler.name, str(compiler.version) - ) - if not os.path.isdir(modulepath_save): - os.makedirs(modulepath_save) - logging.info(" ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) - MODULEPATHS_SAVE.append(modulepath_save) - - # For tcl modules remove the compiler/mpi prefices from the module contents - if module_choice == "tcl": - logging.info( - " ... ... removing compiler/mpi prefices from tcl modulefiles in {}".format( - modulepath_save - ) - ) - for root, ddir, files in os.walk(modulepath_save): - for ffile in files: - filepath = os.path.join(root, ffile) - logging.debug( - " ... ... ... removing compiler/mpi prefices in {}".format( - filepath - ) - ) - # First, compiler-only dependent modules - # These can depend on other compilers than the - # compiler this MPI was built with, loop over all - for pattern in module_replace_patterns: - for tmp_compiler in mock_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") - cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( - pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) - # Then, compiler+mpi-dependent modules - # By definition, these can only depend on the compiler that - # this MPI was built with - no need to loop over all compilers - for pattern in module_replace_patterns: - for tmp_compiler in mock_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") - cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format( - pattern, - mpi_provider.name, - mpi_provider.version, - tmp_compiler_name, - tmp_compiler_version, - filepath, - sed_syntax_fix, - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) - - # The remainder of the loop is only needed for the preferred compiler - if not compiler.name in preferred_compiler: - continue - - # Path and name for mpi module file - mpi_module_dir = os.path.join( - module_dir, compiler.name, str(compiler.version), "stack-" + mpi_provider.name - ) - mpi_module_file = os.path.join( - mpi_module_dir, str(mpi_provider.version).split("-")[0] + MODULE_FILE_EXTENSION[module_choice] - ) - - substitutes = SUBSTITUTES_TEMPLATE.copy() - # - if mpi_provider.external: - # Check if the external package has a prefix - if mpi_provider.prefix: - prefix = mpi_provider.prefix - # Workaround for intel-oneapi-mpi. Because of the way this package - # is configured in spack, the prefix may be incomplete - if mpi_provider.name == "intel-oneapi-mpi": - if not os.path.isdir(os.path.join(prefix, "bin")): - prefix = os.path.join(prefix, "mpi", str(mpi_provider.version)) - if not os.path.isdir(os.path.join(prefix, "bin")): - raise Exception("Invalid prefix for external MPI provider: {prefix}") - # Replace "-" in mpi_provider.name with "_" for _ROOT environment variable - substitutes["MPIROOT"] = setenv_command( - module_choice, mpi_provider.name.replace("-", "_") + "_ROOT", prefix - ) - logging.debug( - " ... ... MPIROOT: {}".format(substitutes["MPIROOT"]).rstrip("\n") - ) - else: - prefix = None - - # Check if the external package has modules - if mpi_provider.external_modules: - raise Exception(f"UNTESTED XYZ MPI MODULES: {mpi_provider.external_modules}") - # Existing non-spack modules to load - for module in mpi_provider.external_modules: - substitutes["MODULELOADS"] += module_load_command( - module_choice, module - ) - substitutes["MODULEPREREQS"] += module_prereq_command( - module_choice, module - ) - substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") - substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") - logging.debug( - " ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"]) - ) - logging.debug( - " ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"]) - ) - # If no modules, then the external package must have a prefix - elif prefix: - # PATH and compiler wrapper environment variables - bindir = os.path.join(prefix, "bin") - if os.path.isdir(bindir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "PATH", bindir - ) - # LD_LIBRARY_PATH AND PKG_CONFIG_PATH - libdir = os.path.join(prefix, "lib") - if os.path.isdir(libdir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "LD_LIBRARY_PATH", libdir - ) - pkgconfigdir = os.path.join(libdir, "pkgconfig") - if os.path.isdir(pkgconfigdir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "PKG_CONFIG_PATH", pkgconfigdir - ) - lib64dir = os.path.join(prefix, "lib64") - if os.path.isdir(lib64dir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "LD_LIBRARY_PATH", lib64dir - ) - pkgconfig64dir = os.path.join(lib64dir, "pkgconfig") - if os.path.isdir(pkgconfig64dir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "PKG_CONFIG_PATH", pkgconfig64dir - ) - # MANPATH - mandir = os.path.join(prefix, "share/man") - if os.path.isdir(mandir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "MANPATH", mandir - ) - # ACLOCAL_PATH - aclocaldir = os.path.join(prefix, "share/aclocal") - if os.path.isdir(aclocaldir): - substitutes["ENVVARS"] += prepend_path_command( - module_choice, "ACLOCAL_PATH", aclocaldir - ) - else: - raise Exception(f"Neither prefix nor modules configured for external MPI provider {mpi_provider}") - else: - # Simple workaround for now - depend on spack building the mpi module - # and load it with the wrapper, don't set any of the other substitutes - module = "{}/{}".format(mpi_provider.name, mpi_provider.version) - substitutes["MODULELOADS"] += module_load_command( - module_choice, module.split("-")[0] - ).rstrip("\n") - substitutes["MODULEPREREQS"] += module_prereq_command( - module_choice, module.split("-")[0] - ).rstrip("\n") - logging.debug( - " ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"]) - ) - logging.debug( - " ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"]) - ) - - # Compiler wrapper environment variables - if mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers": - substitutes["MPICC"] = os.path.join("mpiicx") - substitutes["MPICXX"] = os.path.join("mpiicpx") - if "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and not "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: - substitutes["MPIF77"] = os.path.join("mpiifx") - substitutes["MPIF90"] = os.path.join("mpiifx") - elif not "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: - substitutes["MPIF77"] = os.path.join("mpiifort") - substitutes["MPIF90"] = os.path.join("mpiifort") - else: - raise Exception(f"For {mpi_provider.name}, cannot determine MPI wrapper from FC={COMPILER_SUBSTITUTES_SAVE['FC']}") - elif mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers-classic": - substitutes["MPICC"] = os.path.join("mpiicc") - substitutes["MPICXX"] = os.path.join("mpiicpc") - substitutes["MPIF77"] = os.path.join("mpiifort") - substitutes["MPIF90"] = os.path.join("mpiifort") - else: - substitutes["MPICC"] = os.path.join("mpicc") - substitutes["MPICXX"] = os.path.join("mpic++") - substitutes["MPIF77"] = os.path.join("mpif77") - substitutes["MPIF90"] = os.path.join("mpif90") - # Also set the direct compiler environment variables - substitutes["CC"] = COMPILER_SUBSTITUTES_SAVE["CC"] - substitutes["CXX"] = COMPILER_SUBSTITUTES_SAVE["CXX"] - substitutes["F77"] = COMPILER_SUBSTITUTES_SAVE["F77"] - substitutes["FC"] = COMPILER_SUBSTITUTES_SAVE["FC"] - - # Spack mpi+compiler module hierarchy - append all saved modulepaths - for modulepath in MODULEPATHS_SAVE: - substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) - substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") - logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) - - # Read compiler lua template into module_content string - with open(MPI_TEMPLATES[module_choice]) as f: - module_content = f.read() - - # Substitute variables in module_content - for key in substitutes.keys(): - module_content = module_content.replace( - "@{}@".format(key), substitutes[key] - ) - - # Write mpi lua module - if not os.path.isdir(mpi_module_dir): - os.makedirs(mpi_module_dir) - with open(mpi_module_file, "w") as f: - f.write(module_content) - logging.info(" ... writing {}".format(mpi_module_file)) - - del COMPILER_SUBSTITUTES_SAVE - del MODULEPATHS_SAVE - - logging.info("Metamodule generation completed successfully in {}".format(meta_module_dir)) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250801 b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250801 deleted file mode 100755 index 9ec04f110..000000000 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py.backup.20250801 +++ /dev/null @@ -1,687 +0,0 @@ -#!/usr/bin/env python3 - -import copy -import logging -import os -import re -import sys - -import spack -import spack.environment as ev -# import spack.repo -from spack.provider_index import ProviderIndex - -# logging.basicConfig(level=logging.INFO) -logging.basicConfig(format="%(message)s", level=logging.DEBUG) - -# Get basic directory information -logging.info("Configuring basic directory information ...") -this_script_dir = os.path.realpath(os.path.split(__file__)[0]) -base_dir = os.path.realpath(os.path.join(this_script_dir, "..")) -spack_dir = spack.paths.spack_root -logging.info(" ... script directory: {}".format(this_script_dir)) -logging.info(" ... base directory: {}".format(base_dir)) -logging.info(" ... spack directory: {}".format(spack_dir)) - -# Templates for creating compiler modules -COMPILER_TEMPLATES = { - "lmod": os.path.join(this_script_dir, "templates/compiler.lua"), - "tcl": os.path.join(this_script_dir, "templates/compiler"), -} -MPI_TEMPLATES = { - "lmod": os.path.join(this_script_dir, "templates/mpi.lua"), - "tcl": os.path.join(this_script_dir, "templates/mpi"), -} -MODULE_FILE_EXTENSION = { - "lmod": ".lua", - "tcl": "" -} - -SUBSTITUTES_TEMPLATE = { - "MODULELOADS": "", - "MODULEPREREQS": "", - "MODULEPATHS": "", - "CC": "", - "CXX": "", - "F77": "", - "FC": "", - "COMPFLAGS": "", - "ENVVARS": "", - "MPICC": "", - "MPICXX": "", - "MPIF77": "", - "MPIROOT": "", -} - - -COMPILER_TRANSLATION_TABLE = { - "gcc" : "gcc", - "clang" : "llvm", - "intel" : "intel-oneapi-compilers-classic", - "oneapi" : "intel-oneapi-compilers", -} - - -def setenv_command(module_choice, key, value): - if module_choice == "lmod": - return 'setenv("{}", "{}")\n'.format(key, value) - else: - return "setenv {{{}}} {{{}}}\n".format(key, value) - - -def unsetenv_command(module_choice, key, value): - if module_choice == "lmod": - return 'unsetenv("{}", "{}")\n'.format(key, value) - else: - return "unsetenv {{{}}} {{{}}}\n".format(key, value) - - -def append_path_command(module_choice, key, value): - if module_choice == "lmod": - return 'append_path("{}", "{}")\n'.format(key, value) - else: - return "append-path {{{}}} {{{}}}\n".format(key, value) - - -def prepend_path_command(module_choice, key, value): - if module_choice == "lmod": - return 'prepend_path("{}", "{}")\n'.format(key, value) - else: - return "prepend-path {{{}}} {{{}}}\n".format(key, value) - - -def module_load_command(module_choice, module): - if module_choice == "lmod": - return 'load("{}")\n'.format(module) - else: - return """if {{ [ module-info mode load ] && ![ is-loaded {0} ] }} {{ - module load {0} -}}\n""".format( - module - ) - - -def module_prereq_command(module_choice, module): - if module_choice == "lmod": - return 'prereq("{}")\n'.format(module) - else: - return "" - - -def modulepath_prepend_command(module_choice, modulepath): - if module_choice == "lmod": - return 'prepend_path("MODULEPATH", "{}")\n'.format(modulepath) - else: - return "prepend-path {{MODULEPATH}} {{{}}}\n".format(modulepath) - - -def substitute_config_vars(config_str): - """ - Substitute spack-specific and environment variables that may be present - in configuration files. See: - https://spack.readthedocs.io/en/latest/configuration.html#config-file-variables - """ - spack_vars = { - "ENV": ev.active_environment().path, - "SPACK": os.getenv("SPACK_ROOT"), - "TEMPDIR": None, - "USER": os.getenv("HOME"), - "USER_CACHE_PATH": os.path.join(os.getenv("HOME"), ".spack"), - } - - if config_str.startswith("~"): - config_str = config_str.replace("~", os.getenv("HOME")) - - # Get var as it appears in the string (e.g. ${env}), and its name (e.g. env) - matches = re.findall(r"(\$(\w+))|(\${(\w+)})", config_str) - for match in matches: - if match[0]: - pair = (match[0], match[1]) - else: - pair = (match[2], match[3]) - - var_string = pair[0] - var_name = pair[1].upper() - - sub_value = spack_vars[var_name] if spack_vars[var_name] else os.getenv(var_name) - config_str = config_str.replace(var_string, sub_value) - - return config_str - - -def setup_meta_modules(): - # Find currently active spack environment, activate here - logging.info("Configuring active spack environment ...") - env_dir = ev.active_environment().path - if not env_dir: - raise Exception("No active spack environment") - env = spack.environment.Environment(env_dir) - spack.environment.environment.activate(env) - logging.info(" ... environment directory: {}".format(env_dir)) - - # Parse spack main config from environment - logging.info("Parsing spack environment main config ...") - main_config = spack.config.get("config") - install_dir = substitute_config_vars(main_config["install_tree"]["root"]) - - if not os.path.isabs(install_dir): - install_dir = os.path.realpath(os.path.join(env_dir, install_dir)) - else: - install_dir = os.path.realpath(install_dir) - logging.info(" ... install directory: {}".format(install_dir)) - - # Parse spack module config from environment - logging.info("Parsing spack environment modules config ...") - module_config = spack.config.get("modules") - - # Check which modules are used - tcl or lmod (can only be one) - if len(module_config["default"]["enable"]) > 1: - raise Exception("Can use either lmod or tcl modules, not both") - module_choice = module_config["default"]["enable"][0] - logging.info(" ... configured to use {} modules".format(module_choice)) - - # Need to set a few variables when tcl modules are used - if module_choice == "tcl": - module_replace_patterns = ["is-loaded", "module load", "depends-on"] - # sed syntax differs on macOS - if sys.platform == "darwin": - sed_syntax_fix = "''" - else: - sed_syntax_fix = "" - - # Top-level module directory - module_dir = substitute_config_vars(module_config["default"]["roots"][module_choice]) - if not os.path.isabs(module_dir): - module_dir = os.path.realpath(os.path.join(env_dir, module_dir)) - else: - module_dir = os.path.realpath(module_dir) - logging.info(f" ... module directory: {module_dir}") - - # Get all specs and determine compilers and mpi providers - hashes = env.all_hashes() - specs = spack.store.STORE.db.query(hashes=hashes) - q = ProviderIndex(specs=specs, repository=spack.repo.PATH) - - c_providers = q.providers_for("c") - cxx_providers = q.providers_for("cxx") - fortran_providers = q.providers_for("fortran") - compilers = list(set(c_providers + cxx_providers + fortran_providers)) - if not compilers: - raise Exception("No compilers found") - logging.info(f" ... compilers: {compilers}") - - # To remove compiler prefices from tcl modulefiles, we need - # a mock compiler "none@none" for external packages. Also - # need a list for lmod to check core compiler - #if module_choice == "tcl": - mock_compiler_list = [x.name+"@"+str(x.version) for x in compilers] + ["none@none"] - logging.debug(f" ... mock_compiler_list: {mock_compiler_list}") - - mpi_providers = q.providers_for("mpi") - if not len(mpi_providers)==1: - raise Exception(f"Expected exactly one MPI provider, but got {mpi_providers}") - logging.info(f" ... mpi_providers: {mpi_providers}") - - # Core compilers is only a valid options for lmod - if module_choice == "lmod": - # Determine core compiler(s) and make sure they are not used - core_compilers = module_config["default"][module_choice]["core_compilers"] - logging.info(" ... core compilers: {}".format(core_compilers)) - # Check that none of the compilers used for the stack is a core compiler - for core_compiler in core_compilers: - if any(core_compiler in x for x in mock_compiler_list): - raise Exception( - """Not supported: compiler used for environment - is in list of core compilers""" - ) - - # Prepare meta module directory - logging.info("Preparing meta module directory ...") - meta_module_dir = os.path.join(module_dir, "Core") - if not os.path.isdir(meta_module_dir): - os.mkdir(meta_module_dir) - logging.info(" ... meta module directory : {}".format(meta_module_dir)) - - - # DH* 20250730 - DO WE NEED BOTH WAYS TO DETERMINE THE PREFERRED COMPILER? - # MAYBE KEEP FOR NOW UNTIL WE KNOW BETTER WHAT IS NEEDED AND WHAT NOT. - - # Determine the preferred compiler and sort the list of compilers so that - # the preferred compiler comes last. This is so that all other compilers - # populate the MODULEPATHS_SAVE list before the preferred compiler - # takes it and adds it to the stack-COMPILER metamodule. Likewise, we need - # to save the list of compiler substitutions from the preferred compiler - # so that we have access to it when we build the MPI meta module. - try: - preferred_compilers = spack.config.get("packages")["all"]["require"] - except: - raise Exception( - """Unable to detect preferred compiler from environment. - Does the environment have the config entry 'packages:all:require?'""" - ) - if len(preferred_compilers)>1: - raise Exception(f"Invalid value for packages:all:require is {preferred_compilers}") - match = re.search(r'%\[when=%fortran\](\S+)', preferred_compilers[0]) - if match and match.group(1) in COMPILER_TRANSLATION_TABLE.keys(): - preferred_compiler_v1 = COMPILER_TRANSLATION_TABLE[match.group(1)] - else: - raise Exception(f"Invalid value for packages:all:require {preferred_compilers[0]}") - - try: - preferred_compilers = spack.config.get("packages")["fortran"]["require"] - except: - raise Exception( - """Unable to detect preferred compiler from environment. - Does the environment have the config entry 'packages:fortran:require?'""" - ) - if len(preferred_compilers)>1: - raise Exception(f"Invalid value for packages:fortran:require is {preferred_compilers}") - if preferred_compilers[0] in COMPILER_TRANSLATION_TABLE.values(): - preferred_compiler_v2 = preferred_compilers[0] - else: - raise Exception(f"Invalid value for packages:fortran:require {preferred_compilers[0]}") - if preferred_compiler_v1 == preferred_compiler_v2: - preferred_compiler = preferred_compiler_v1 - del preferred_compilers - del preferred_compiler_v1 - del preferred_compiler_v2 - else: - raise Exception(f"Multiple preferred compilers in spack config: {preferred_compiler_v1} vs {preferred_compiler_v2}") - logging.info(" ... preferred compiler: {}".format(preferred_compiler)) - # *DH 20250730 - - # Sort the list using a custom key - def custom_sort_key(entry): - # Return a tuple where the first element is 1 if the entry contains the word, else 0 - # The second element is the entry itself for natural sorting within groups - return (1 if preferred_compiler in entry else 0, entry) - compilers = sorted(compilers, key=custom_sort_key) - - # Create compiler modules - logging.info("Creating compiler modules ...") - - # Collect and save modulepaths for the preferred compiler - MODULEPATHS_SAVE = [] - - # Append modulepath for external specs and specs without compiler dependencies - modulepath_save = os.path.join(module_dir, "none", "none") - if not os.path.isdir(modulepath_save): - os.makedirs(modulepath_save) - logging.info(" ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) - MODULEPATHS_SAVE.append(modulepath_save) - # For tcl modules remove the compiler prefices from the module contents - if module_choice == "tcl": - logging.info( - " ... ... removing compiler prefices from tcl modulefiles in {}".format( - modulepath_save - ) - ) - - for root, ddir, files in os.walk(modulepath_save): - for ffile in files: - filepath = os.path.join(root, ffile) - logging.debug( - " ... ... ... removing compiler prefices in {}".format(filepath) - ) - for pattern in module_replace_patterns: - for tmp_compiler in mock_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") - cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( - pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) - - # Initialize saved substitutes to None (populate for preferred compiler later) - COMPILER_SUBSTITUTES_SAVE = None - - for compiler in compilers: - logging.info(f" ... configuring compiler {compiler.name}@{compiler.version}") - - modulepath_save = os.path.join(module_dir, compiler.name, str(compiler.version)) - if not os.path.isdir(modulepath_save): - os.makedirs(modulepath_save) - logging.info(" ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) - MODULEPATHS_SAVE.append(modulepath_save) - - # For tcl modules remove the compiler prefices from the module contents - if module_choice == "tcl": - logging.info( - " ... ... removing compiler prefices from tcl modulefiles in {}".format( - modulepath_save - ) - ) - - for root, ddir, files in os.walk(modulepath_save): - for ffile in files: - filepath = os.path.join(root, ffile) - logging.debug( - " ... ... ... removing compiler prefices in {}".format(filepath) - ) - for pattern in module_replace_patterns: - for tmp_compiler in mock_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") - cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( - pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) - - # The remainder of the loop is only needed for the preferred compiler - if not compiler.name in preferred_compiler: - continue - - logging.info( - " ... configuring stack compiler {}@{}".format(compiler.name, compiler.version) - ) - compiler_module_dir = os.path.join(meta_module_dir, "stack-" + compiler.name) - compiler_module_file = os.path.join( - compiler_module_dir, str(compiler.version) + MODULE_FILE_EXTENSION[module_choice] - ) - substitutes = SUBSTITUTES_TEMPLATE.copy() - - if not compiler.external: - raise Exception(f"spack-built compiler {compiler} not yet supported") - - # Use existing modules for external mpi providers; otherwise, use spack-built module - if compiler.external and compiler.external_modules: - for module in compiler.external_modules: - substitutes["MODULELOADS"] += module_load_command(module_choice, module) - substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) - else: - module = "{}/{}".format(compiler.name, compiler.version) - substitutes["MODULELOADS"] += module_load_command(module_choice, module) - substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) - substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") - substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") - logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) - logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) - - # Compiler environment variables; names are lowercase in spack - substitutes["CC"] = compiler.extra_attributes["compilers"]["c"] - substitutes["CXX"] = compiler.extra_attributes["compilers"]["cxx"] - substitutes["F77"] = compiler.extra_attributes["compilers"]["fortran"] - substitutes["FC"] = compiler.extra_attributes["compilers"]["fortran"] - logging.debug(" ... ... CC : {}".format(substitutes["CC"])) - logging.debug(" ... ... CXX : {}".format(substitutes["CXX"])) - logging.debug(" ... ... F77 : {}".format(substitutes["F77"])) - logging.debug(" ... ... FC' : {}".format(substitutes["FC"])) - - # Compiler flags; names are lowercase in spack - if "flags" in compiler.extra_attributes.keys(): - for flag_name in compiler.extra_attributes["flags"].keys(): - flag_values = compiler["compiler"]["flags"][flag_name] - substitutes["COMPFLAGS"] += setenv_command( - module_choice, flag_name.upper(), flag_values - ) - substitutes["COMPFLAGS"] = substitutes["COMPFLAGS"].rstrip("\n") - logging.debug(" ... ... COMPFLAGS: {}".format(substitutes["COMPFLAGS"])) - - # Environment variables; case-sensitive in spack - if "environment" in compiler.extra_attributes.keys(): - # DH* CAN WE SIMPLIFY THIS WITH A LOOP AND A GENERIC FUNCTION? - # append_path - if "append_path" in compiler.extra_attributes["environment"].keys(): - for env_name in compiler.extra_attributes["environment"]["append_path"]: - env_values = compiler.extra_attributes["environment"]["append_path"][env_name] - substitutes["ENVVARS"] += append_path_command( - module_choice, env_name, env_values - ) - # prepend_path - if "prepend_path" in compiler.extra_attributes["environment"].keys(): - for env_name in compiler.extra_attributes["environment"]["prepend_path"]: - env_values = compiler.extra_attributes["environment"]["prepend_path"][env_name] - substitutes["ENVVARS"] += prepend_path_command( - module_choice, env_name, env_values - ) - # set - if "set" in compiler.extra_attributes["environment"].keys(): - for env_name in compiler.extra_attributes["environment"]["set"]: - env_values = compiler.extra_attributes["environment"]["set"][env_name] - substitutes["ENVVARS"] += setenv_command( - module_choice, env_name, env_values - ) - # unset - if "unset" in compiler.extra_attributes["environment"].keys(): - for env_name in compiler.extra_attributes["environment"]["unset"]: - env_values = compiler.extra_attributes["environment"]["unset"][env_name] - substitutes["ENVVARS"] += unsetenv_command( - module_choice, env_name, env_values - ) - # *DH - substitutes["ENVVARS"] = substitutes["ENVVARS"].rstrip("\n") - logging.debug(" ... ... ENVVARS : {}".format(substitutes["ENVVARS"])) - - # Spack compiler module hierarchy - append all saved modulepaths - for modulepath in MODULEPATHS_SAVE: - substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) - substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") - logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) - - # Read compiler template into module_content string - with open(COMPILER_TEMPLATES[module_choice]) as f: - module_content = f.read() - - # Substitute variables in module_content - for key in substitutes.keys(): - module_content = module_content.replace("@{}@".format(key), substitutes[key]) - - # Write compiler lua module - if not os.path.isdir(compiler_module_dir): - os.makedirs(compiler_module_dir) - with open(compiler_module_file, "w") as f: - f.write(module_content) - logging.info(" ... writing {}".format(compiler_module_file)) - # If this is the last compiler in the list (i.e. the preferred compiler), - # then save the substitutes for later use for building the MPI meta module. - if compiler == compilers[-1]: - COMPILER_SUBSTITUTES_SAVE = substitutes - - del MODULEPATHS_SAVE - - # Collect and save modulepaths for MPI with preferred compiler - MODULEPATHS_SAVE = [] - - # Create mpi modules - as of July 2025, only one - # MPI provider is allowed (for the preferred compiler) - for mpi_provider in mpi_providers: - - # Append modulepath for external specs and specs without compiler dependencies - modulepath_save = os.path.join(module_dir, mpi_provider.name, str(mpi_provider.version), "none", "none") - if not os.path.isdir(modulepath_save): - os.makedirs(modulepath_save) - logging.info(" ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) - MODULEPATHS_SAVE.append(modulepath_save) - - # For tcl modules remove the compiler/mpi prefices from the module contents - if module_choice == "tcl": - logging.info( - " ... ... removing no-compiler/mpi prefices from tcl modulefiles in {}".format( - modulepath_save - ) - ) - for root, ddir, files in os.walk(modulepath_save): - for ffile in files: - filepath = os.path.join(root, ffile) - logging.debug( - " ... ... ... removing no-compiler/mpi prefices in {}".format( - filepath - ) - ) - # First, compiler-only dependent modules - # These can depend on other compilers than the - # compiler this MPI was built with, loop over all - for pattern in module_replace_patterns: - for tmp_compiler in mock_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") - cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( - pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) - # Then, compiler+mpi-dependent modules - # By definition, these can only depend on the compiler that - # this MPI was built with - no need to loop over all compilers - for pattern in module_replace_patterns: - for tmp_compiler in mock_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") - cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format( - pattern, - mpi_provider.name, - mpi_provider.version, - tmp_compiler_name, - tmp_compiler_version, - filepath, - sed_syntax_fix, - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) - - for compiler in compilers: - logging.info( - " ... configuring stack mpi library {}@{} for compiler {}@{}".format( - mpi_provider.name, mpi_provider.version, compiler.name, compiler.version - ) - ) - - # Spack mpi+compiler module hierarchy - modulepath_save = os.path.join( - module_dir, mpi_provider.name, str(mpi_provider.version), compiler.name, str(compiler.version) - ) - if not os.path.isdir(modulepath_save): - os.makedirs(modulepath_save) - logging.info(" ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) - MODULEPATHS_SAVE.append(modulepath_save) - - # For tcl modules remove the compiler/mpi prefices from the module contents - if module_choice == "tcl": - logging.info( - " ... ... removing compiler/mpi prefices from tcl modulefiles in {}".format( - modulepath_save - ) - ) - for root, ddir, files in os.walk(modulepath_save): - for ffile in files: - filepath = os.path.join(root, ffile) - logging.debug( - " ... ... ... removing compiler/mpi prefices in {}".format( - filepath - ) - ) - # First, compiler-only dependent modules - # These can depend on other compilers than the - # compiler this MPI was built with, loop over all - for pattern in module_replace_patterns: - for tmp_compiler in mock_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") - cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( - pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) - # Then, compiler+mpi-dependent modules - # By definition, these can only depend on the compiler that - # this MPI was built with - no need to loop over all compilers - for pattern in module_replace_patterns: - for tmp_compiler in mock_compiler_list: - (tmp_compiler_name, tmp_compiler_version) = tmp_compiler.split("@") - cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format( - pattern, - mpi_provider.name, - mpi_provider.version, - tmp_compiler_name, - tmp_compiler_version, - filepath, - sed_syntax_fix, - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) - - # The remainder of the loop is only needed for the preferred compiler - if not compiler.name in preferred_compiler: - continue - - # Path and name for mpi module file - mpi_module_dir = os.path.join( - module_dir, compiler.name, str(compiler.version), "stack-" + mpi_provider.name - ) - mpi_module_file = os.path.join( - mpi_module_dir, str(mpi_provider.version).split("-")[0] + MODULE_FILE_EXTENSION[module_choice] - ) - - substitutes = SUBSTITUTES_TEMPLATE.copy() - # Use existing modules for external mpi providers; otherwise, use spack-built module - if mpi_provider.external and mpi_provider.external_modules: - for module in mpi_provider.external_modules: - substitutes["MODULELOADS"] += module_load_command(module_choice, module) - substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) - else: - module = "{}/{}".format(mpi_provider.name, mpi_provider.version) - substitutes["MODULELOADS"] += module_load_command(module_choice, module) - substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) - substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") - substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") - logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) - logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) - - # Compiler wrapper environment variables - if mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers": - substitutes["MPICC"] = os.path.join("mpiicx") - substitutes["MPICXX"] = os.path.join("mpiicpx") - if "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and not "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: - substitutes["MPIF77"] = os.path.join("mpiifx") - substitutes["MPIF90"] = os.path.join("mpiifx") - elif not "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: - substitutes["MPIF77"] = os.path.join("mpiifort") - substitutes["MPIF90"] = os.path.join("mpiifort") - else: - raise Exception(f"For {mpi_provider.name}, cannot determine MPI wrapper from FC={COMPILER_SUBSTITUTES_SAVE['FC']}") - elif mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers-classic": - substitutes["MPICC"] = os.path.join("mpiicc") - substitutes["MPICXX"] = os.path.join("mpiicpc") - substitutes["MPIF77"] = os.path.join("mpiifort") - substitutes["MPIF90"] = os.path.join("mpiifort") - else: - substitutes["MPICC"] = os.path.join("mpicc") - substitutes["MPICXX"] = os.path.join("mpic++") - substitutes["MPIF77"] = os.path.join("mpif77") - substitutes["MPIF90"] = os.path.join("mpif90") - # Also set the direct compiler environment variables - substitutes["CC"] = COMPILER_SUBSTITUTES_SAVE["CC"] - substitutes["CXX"] = COMPILER_SUBSTITUTES_SAVE["CXX"] - substitutes["F77"] = COMPILER_SUBSTITUTES_SAVE["F77"] - substitutes["FC"] = COMPILER_SUBSTITUTES_SAVE["FC"] - - # Spack mpi+compiler module hierarchy - append all saved modulepaths - for modulepath in MODULEPATHS_SAVE: - substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) - substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") - logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) - - # Read compiler lua template into module_content string - with open(MPI_TEMPLATES[module_choice]) as f: - module_content = f.read() - - # Substitute variables in module_content - for key in substitutes.keys(): - module_content = module_content.replace( - "@{}@".format(key), substitutes[key] - ) - - # Write mpi lua module - if not os.path.isdir(mpi_module_dir): - os.makedirs(mpi_module_dir) - with open(mpi_module_file, "w") as f: - f.write(module_content) - logging.info(" ... writing {}".format(mpi_module_file)) - - del COMPILER_SUBSTITUTES_SAVE - del MODULEPATHS_SAVE - - logging.info("Metamodule generation completed successfully in {}".format(meta_module_dir)) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi index ce0cedcfe..8a7ecbed1 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi @@ -22,17 +22,23 @@ setenv {MPICC} {@MPICC@} setenv {MPICXX} {@MPICXX@} setenv {MPIF77} {@MPIF77@} setenv {MPIF90} {@MPIF90@} -setenv {MPI_CC} {@MPICC@} -setenv {MPI_CXX} {@MPICXX@} -setenv {MPI_F77} {@MPIF77@} -setenv {MPI_F90} {@MPIF90@} -# intel specific mpi wrapper environment variables +# underlying compilers for mpi distributions setenv {I_MPI_CC} {@CC@} setenv {I_MPI_CXX} {@CXX@} setenv {I_MPI_F77} {@F77@} setenv {I_MPI_F90} {@FC@} setenv {I_MPI_FC} {@FC@} +setenv {OMPI_CC} {@CC@} +setenv {OMPI_CXX} {@CXX@} +setenv {OMPI_F77} {@F77@} +setenv {OMPI_F90} {@FC@} +setenv {OMPI_FC} {@FC@} +setenv {MPICH_CC} {@CC@} +setenv {MPICH_CXX} {@CXX@} +setenv {MPICH_F77} {@F77@} +setenv {MPICH_F90} {@FC@} +setenv {MPICH_FC} {@FC@} # compiler flags and other environment variables @COMPFLAGS@ diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua index ec841f2d0..d0aea6e95 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua @@ -31,12 +31,22 @@ setenv("MPI_CXX", "@MPICXX@") setenv("MPI_F77", "@MPIF77@") setenv("MPI_F90", "@MPIF90@") --- intel specific mpi wrapper environment variables +# underlying compilers for mpi distributions setenv("I_MPI_CC", "@CC@") setenv("I_MPI_CXX", "@CXX@") setenv("I_MPI_F77", "@F77@") setenv("I_MPI_F90", "@FC@") setenv("I_MPI_FC", "@FC@") +setenv("OMPI_CC", "@CC@") +setenv("OMPI_CXX", "@CXX@") +setenv("OMPI_F77", "@F77@") +setenv("OMPI_F90", "@FC@") +setenv("OMPI_FC", "@FC@") +setenv("MPICH_CC", "@CC@") +setenv("MPICH_CXX", "@CXX@") +setenv("MPICH_F77", "@F77@") +setenv("MPICH_F90", "@FC@") +setenv("MPICH_FC", "@FC@") -- compiler flags and other environment variables @COMPFLAGS@ From 08ba848f9bd0303a77660328f89d86317dbb07c8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 29 Sep 2025 11:09:24 -0600 Subject: [PATCH 120/163] [skip ci] Update site config for Blueback (add Oneapi 2025.2.1) --- configs/sites/tier1/blueback/packages.yaml | 21 ++++---- .../tier1/blueback/packages_gcc-13.3.0.yaml | 18 +++---- .../blueback/packages_oneapi-2025.2.1.yaml | 52 +++++++++++++++++++ .../sites/tier1/blueback/packages_oneapi.yaml | 35 ------------- .../spack-stack/stack/meta_modules.py | 3 +- 5 files changed, 72 insertions(+), 57 deletions(-) create mode 100644 configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml delete mode 100644 configs/sites/tier1/blueback/packages_oneapi.yaml diff --git a/configs/sites/tier1/blueback/packages.yaml b/configs/sites/tier1/blueback/packages.yaml index 440fa0ba0..88ceccb47 100644 --- a/configs/sites/tier1/blueback/packages.yaml +++ b/configs/sites/tier1/blueback/packages.yaml @@ -1,19 +1,9 @@ packages: - -# Modification of common packages - # Use system zlib instead of spack-built zlib-ng - all: - providers: - zlib-api:: [zlib] + # Modification of common packages zlib-api: buildable: False - zlib: - buildable: False - externals: - - spec: zlib@1.2.13 - prefix: /usr -# All other packages listed alphabetically + # All other packages listed alphabetically autoconf: externals: - spec: autoconf@2.69 @@ -51,10 +41,12 @@ packages: - spec: gettext@0.20.2 prefix: /usr git: + buildable: False externals: - spec: git@2.35.3+tcltk prefix: /usr git-lfs: + buildable: False externals: - spec: git-lfs@3.3.0 prefix: /usr @@ -117,3 +109,8 @@ packages: externals: - spec: wget@1.20.3 prefix: /usr + zlib: + buildable: False + externals: + - spec: zlib@1.2.13 + prefix: /usr diff --git a/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml index d7f89edfe..876c2b482 100644 --- a/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml +++ b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml @@ -1,31 +1,31 @@ packages: mpi: - buildable: False + buildable: false require: - - "cray-mpich@8.1.32" + - cray-mpich@8.1.32 gcc: - buildable: False + buildable: false externals: - spec: gcc@13.3.0 languages:='c,c++,fortran' prefix: /usr modules: - PrgEnv-gnu/8.6.0 - gcc-native/13.2 - - libfabric/1.22.0 extra_attributes: compilers: c: /usr/bin/gcc cxx: /usr/bin/g++ fortran: /usr/bin/gfortran environment: - prepend_path: - LD_LIBRARY_PATH: '/opt/cray/libfabric/1.22.0/lib64' set: CONFIG_SITE: '' - CRAY_CPU_TARGET: x86-genoa + PE_GCC_EXTERNAL: native cray-mpich: + buildable: false externals: - - spec: cray-mpich@8.1.32%gcc@13.3.0 ~wrappers + - spec: cray-mpich@8.1.32 + prefix: /opt/cray/pe/mpich/8.1.32/ofi/GNU/12.3 modules: - - cray-mpich/8.1.32 + - libfabric/1.22.0 - craype-network-ofi + - cray-mpich/8.1.32 diff --git a/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml b/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml new file mode 100644 index 000000000..d11efa8ad --- /dev/null +++ b/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml @@ -0,0 +1,52 @@ +packages: + mpi: + buildable: false + require: + - cray-mpich@8.1.32 + intel-oneapi-compilers: + buildable: false + externals: + - spec: intel-oneapi-compilers@2025.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1 + modules: + - intel-oneapi/2025.2.1 + extra_attributes: + compilers: + c: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1/compiler/2025.2/bin/icx + cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1/compiler/2025.2/bin/icpx + fortran: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1/compiler/2025.2/bin/ifx + environment: + set: + CONFIG_SITE: '' + gcc: + buildable: false + externals: + - spec: gcc@13.3.0 languages:='c,c++,fortran' + prefix: /usr + modules: + - PrgEnv-gnu/8.6.0 + - gcc-native/13.2 + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + environment: + set: + CONFIG_SITE: '' + PE_GCC_EXTERNAL: native + cray-mpich: + buildable: false + externals: + - spec: cray-mpich@8.1.32 + prefix: /opt/cray/pe/mpich/8.1.32/ofi/INTEL/2022.1 + modules: + - libfabric/1.22.0 + - craype-network-ofi + - cray-mpich/8.1.32 + - craype-network-ofi + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.2 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1 diff --git a/configs/sites/tier1/blueback/packages_oneapi.yaml b/configs/sites/tier1/blueback/packages_oneapi.yaml deleted file mode 100644 index 98e5c5d56..000000000 --- a/configs/sites/tier1/blueback/packages_oneapi.yaml +++ /dev/null @@ -1,35 +0,0 @@ -packages: - all: - compiler:: [oneapi@2024.2.1, oneapi@2025.0.4, gcc@13.3.0] - providers: - mpi:: [cray-mpich@8.1.32] - mpi: - buildable: False - cray-mpich: - externals: - - spec: cray-mpich@8.1.32~wrappers%oneapi@2024.2.1 - modules: - - cray-mpich/8.1.32 - - craype-network-ofi - - spec: cray-mpich@8.1.32~wrappers%oneapi@2025.0.4 - modules: - - cray-mpich/8.1.32 - - craype-network-ofi - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1 - prefix: /p/app/intel/oneapi-combined - - spec: intel-oneapi-mkl@2025.0%oneapi@2025.0.4 - prefix: /p/app/intel/oneapi-combined - intel-oneapi-tbb: - externals: - - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.1 - prefix: /p/app/intel/oneapi-combined - - spec: intel-oneapi-tbb@2022.0%oneapi@2025.0.4 - prefix: /p/app/intel/oneapi-combined - intel-oneapi-runtime: - externals: - - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 - prefix: /p/app/intel/oneapi-combined - - spec: intel-oneapi-runtime@2025.0.4%oneapi@2025.0.4 - prefix: /p/app/intel/oneapi-combined diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 90665543d..8a4281aa5 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -559,7 +559,8 @@ def custom_sort_key(entry): substitutes["MPIROOT"] += setenv_command(module_choice, "MPI_ROOT", mpi_provider.prefix) substitutes["MPIROOT"] += setenv_command(module_choice, "MPI_HOME", mpi_provider.prefix) substitutes["MPIROOT"] += setenv_command(module_choice, "MPICH_DIR", mpi_provider.prefix) - logging.debug(" ... ... MPIROOT: {}".format(substitutes["MPIROOT"])) + for line in substitutes["MPIROOT"].split('\n'): + logging.debug(" ... ... MPIROOT: {}".format(line)) # Compiler wrapper environment variables if mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers": From 6921c2d90e6f40baf3ede8d837e0d5a17c9ba186 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 29 Sep 2025 13:39:31 -0600 Subject: [PATCH 121/163] Update nautilus site config --- configs/sites/tier1/nautilus/compilers.yaml | 99 ------------------- configs/sites/tier1/nautilus/packages.yaml | 21 ++-- .../sites/tier1/nautilus/packages_aocc.yaml | 35 ------- .../tier1/nautilus/packages_gcc-13.4.0.yaml | 23 +++++ .../sites/tier1/nautilus/packages_gcc.yaml | 14 --- .../sites/tier1/nautilus/packages_intel.yaml | 18 ---- .../nautilus/packages_oneapi-2025.2.1.yaml | 52 ++++++++++ .../sites/tier1/nautilus/packages_oneapi.yaml | 49 --------- 8 files changed, 81 insertions(+), 230 deletions(-) delete mode 100644 configs/sites/tier1/nautilus/compilers.yaml delete mode 100644 configs/sites/tier1/nautilus/packages_aocc.yaml create mode 100644 configs/sites/tier1/nautilus/packages_gcc-13.4.0.yaml delete mode 100644 configs/sites/tier1/nautilus/packages_gcc.yaml delete mode 100644 configs/sites/tier1/nautilus/packages_intel.yaml create mode 100644 configs/sites/tier1/nautilus/packages_oneapi-2025.2.1.yaml delete mode 100644 configs/sites/tier1/nautilus/packages_oneapi.yaml diff --git a/configs/sites/tier1/nautilus/compilers.yaml b/configs/sites/tier1/nautilus/compilers.yaml deleted file mode 100644 index 38c11f160..000000000 --- a/configs/sites/tier1/nautilus/compilers.yaml +++ /dev/null @@ -1,99 +0,0 @@ -compilers:: -- compiler: - spec: aocc@4.0.0 - paths: - cc: /p/app/compilers/amd/aocc/4.0.0/bin/clang - cxx: /p/app/compilers/amd/aocc/4.0.0/bin/clang++ - f77: /p/app/compilers/amd/aocc/4.0.0/bin/flang - fc: /p/app/compilers/amd/aocc/4.0.0/bin/flang - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - slurm - - amd/aocc/4.0.0 - - amd/aocl/aocc/4.0 - environment: {} - extra_rpaths: [] -- compiler: - spec: oneapi@2024.2.1 - paths: - cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icx - cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icpx - f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort - fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - slurm - - tbb/2021.13 - - compiler-rt/2024.2.1 - - compiler/2024.2.1 - - ifort/2024.2.1 - environment: - prepend_path: - PATH: '/opt/rh/gcc-toolset-11/root/usr/bin' - CPATH: '/opt/rh/gcc-toolset-11/root/usr/include' - LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' - MODULEPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles' - append_path: - CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/opt/compiler/include/intel64' - extra_rpaths: [] -- compiler: - spec: oneapi@2025.1.1 - paths: - cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.1.1/compiler/2025.1/bin/icx - cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.1.1/compiler/2025.1/bin/icpx - f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.1.1/compiler/2025.1/bin/ifx - fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.1.1/compiler/2025.1/bin/ifx - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - slurm - - tbb/2022.1 - - compiler-rt/2025.1.1 - - umf/0.10.0 - - compiler-intel-llvm/2025.1.1 - environment: - prepend_path: - PATH: '/opt/rh/gcc-toolset-11/root/usr/bin' - CPATH: '/opt/rh/gcc-toolset-11/root/usr/include' - LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' - MODULEPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2025.1.1/modulefiles' - append_path: - CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2025.1.1/compiler/2025.1/opt/compiler/include/intel64' - extra_rpaths: [] -- compiler: - spec: gcc@11.2.1 - paths: - cc: /opt/rh/gcc-toolset-11/root/usr/bin/gcc - cxx: /opt/rh/gcc-toolset-11/root/usr/bin/g++ - f77: /opt/rh/gcc-toolset-11/root/usr/bin/gfortran - fc: /opt/rh/gcc-toolset-11/root/usr/bin/gfortran - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - slurm - - scl/gcc-toolset-11 - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@13.4.0 - paths: - cc: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/bin/gcc - cxx: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/bin/g++ - f77: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/bin/gfortran - fc: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/bin/gfortran - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - slurm - - gcc/13.4.0 - environment: - prepend_path: - MODULEPATH: '/p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/modulefiles' - extra_rpaths: [] diff --git a/configs/sites/tier1/nautilus/packages.yaml b/configs/sites/tier1/nautilus/packages.yaml index 5e62e0c79..256082a18 100644 --- a/configs/sites/tier1/nautilus/packages.yaml +++ b/configs/sites/tier1/nautilus/packages.yaml @@ -1,21 +1,7 @@ packages: -# Modification of common packages - # Use system zlib instead of spack-built zlib-ng - all: - providers: - zlib-api:: [zlib] + # Modification of common packages zlib-api: buildable: False - zlib: - buildable: False - externals: - - spec: zlib@1.2.11 - prefix: /usr - - # Pin py-torch to 2.5.1 to avoid build errors with 2.6.0 - py-torch: - require: - - '@2.5.1' # All other packages listed alphabetically autoconf: @@ -109,3 +95,8 @@ packages: externals: - spec: wget@1.19.5 prefix: /usr + zlib: + buildable: False + externals: + - spec: zlib@1.2.11 + prefix: /usr diff --git a/configs/sites/tier1/nautilus/packages_aocc.yaml b/configs/sites/tier1/nautilus/packages_aocc.yaml deleted file mode 100644 index 380d51b6e..000000000 --- a/configs/sites/tier1/nautilus/packages_aocc.yaml +++ /dev/null @@ -1,35 +0,0 @@ -packages: -# On Nautilus, use intel-oneapi-mkl as provider -# for blas, lapack, fftw-api with AOCC because of -# problems using openblas in downstream applications -# (e.g. py-pandas): -# ELF load command address/offset not properly aligned" when loading libopenblas.so -# https://github.com/OpenMathLib/OpenBLAS/wiki/Faq#ELFoffset - all: - compiler:: [aocc@4.0.0] - providers: - mpi:: [openmpi@4.1.6] - blas:: [intel-oneapi-mkl] - fftw-api:: [intel-oneapi-mkl] - lapack:: [intel-oneapi-mkl] - mpi: - buildable: False - openmpi: - externals: - - spec: openmpi@4.1.6%aocc@4.0.0~cuda~java~memchecker~static~wrapper-rpath fabrics=ucx schedulers=slurm - prefix: /p/app/penguin/openmpi/4.1.6/aocc-4.0.0 - modules: - - penguin/openmpi/4.1.6/aocc-4.0.0 - - penguin/mpi-vars/aocc - - slurm - openblas: - buildable: False - ectrans: - require:: - - '@1.2.0 +mkl ~fftw' - gsibec: - require:: - - '@1.2.1 +mkl' - py-numpy: - require:: - - '@:1.25 ^intel-oneapi-mkl' diff --git a/configs/sites/tier1/nautilus/packages_gcc-13.4.0.yaml b/configs/sites/tier1/nautilus/packages_gcc-13.4.0.yaml new file mode 100644 index 000000000..150c7e20f --- /dev/null +++ b/configs/sites/tier1/nautilus/packages_gcc-13.4.0.yaml @@ -0,0 +1,23 @@ +packages: + mpi: + buildable: False + require: + - openmpi@5.0.1 + gcc: + externals: + - spec: gcc@13.4.0 languages:='c,c++,fortran' + prefix: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0 + modules: + - gcc/13.4.0 + extra_attributes: + compilers: + c: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/bin/gcc + cxx: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/bin/g++ + fortran: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/bin/gfortran + openmpi: + externals: + - spec: openmpi@5.0.1 + prefix: /p/app/penguin/openmpi/5.0.1/gcc-8.5.0 + modules: + - penguin/openmpi/5.0.1/gcc-8.5.0 + - slurm diff --git a/configs/sites/tier1/nautilus/packages_gcc.yaml b/configs/sites/tier1/nautilus/packages_gcc.yaml deleted file mode 100644 index 2f5f2f4ee..000000000 --- a/configs/sites/tier1/nautilus/packages_gcc.yaml +++ /dev/null @@ -1,14 +0,0 @@ -packages: - all: - compiler:: [gcc@13.4.0] - providers: - mpi:: [openmpi@5.0.1] - mpi: - buildable: False - openmpi: - externals: - - spec: openmpi@5.0.1~cuda~java~memchecker~static~wrapper-rpath fabrics=ucx schedulers=slurm %gcc@13.4.0 - prefix: /p/app/penguin/openmpi/5.0.1/gcc-8.5.0 - modules: - - penguin/openmpi/5.0.1/gcc-8.5.0 - - slurm diff --git a/configs/sites/tier1/nautilus/packages_intel.yaml b/configs/sites/tier1/nautilus/packages_intel.yaml deleted file mode 100644 index 7516471db..000000000 --- a/configs/sites/tier1/nautilus/packages_intel.yaml +++ /dev/null @@ -1,18 +0,0 @@ -packages: - all: - compiler:: [intel@2021.5.0,gcc@11.2.1] - providers: - mpi:: [openmpi@4.1.6] - mpi: - buildable: False - openmpi: - externals: - - spec: openmpi@4.1.6%intel@2021.5.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm - prefix: /p/app/penguin/openmpi/4.1.6/intel-classic-2022.0.2 - modules: - - penguin/openmpi/4.1.6/intel-classic-2022.0.2 - - slurm - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2022.0.2%intel@2021.5.0 - prefix: /p/app/compilers/intel/oneapi diff --git a/configs/sites/tier1/nautilus/packages_oneapi-2025.2.1.yaml b/configs/sites/tier1/nautilus/packages_oneapi-2025.2.1.yaml new file mode 100644 index 000000000..6e7177c3d --- /dev/null +++ b/configs/sites/tier1/nautilus/packages_oneapi-2025.2.1.yaml @@ -0,0 +1,52 @@ +packages: + mpi: + buildable: false + require: + - intel-oneapi-mpi@2021.16 + intel-oneapi-compilers: + buildable: false + externals: + - spec: intel-oneapi-compilers@2025.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1 + modules: + - tbb/2022.2 + - umf/0.11.0 + - compiler-rt/2025.2.1 + - compiler/2025.2.1 + extra_attributes: + compilers: + c: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1/compiler/2025.2/bin/icx + cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1/compiler/2025.2/bin/icpx + fortran: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1/compiler/2025.2/bin/ifx + environment: + prepend_path: + PATH: /opt/rh/gcc-toolset-11/root/usr/bin + CPATH: /opt/rh/gcc-toolset-11/root/usr/include + extra_rpaths: + - /opt/rh/gcc-toolset-11/root/usr/lib64 + - /opt/rh/gcc-toolset-11/root/usr/lib + gcc: + buildable: false + externals: + - spec: gcc@11.2.1 languages:='c,c++,fortran' + prefix: /opt/rh/gcc-toolset-11/root/usr + modules: + - scl/gcc-toolset-11 + extra_attributes: + compilers: + c: /opt/rh/gcc-toolset-11/root/usr/bin/gcc + cxx: /opt/rh/gcc-toolset-11/root/usr/bin/g++ + fortran: /opt/rh/gcc-toolset-11/root/usr/bin/gfortran + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.16 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1 + modules: + - mpi/2021.16 + - slurm + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.2 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1 diff --git a/configs/sites/tier1/nautilus/packages_oneapi.yaml b/configs/sites/tier1/nautilus/packages_oneapi.yaml deleted file mode 100644 index 6c10140da..000000000 --- a/configs/sites/tier1/nautilus/packages_oneapi.yaml +++ /dev/null @@ -1,49 +0,0 @@ -packages: - all: - compiler:: [oneapi@2024.2.1, oneapi@2025.1.1, gcc@11.2.1] - providers: - mpi:: [intel-oneapi-mpi@2021.13, intel-oneapi-mpi@2021.15] - mpi: - buildable: False - intel-oneapi-mpi: - externals: - - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 - modules: - - mpi/2021.13 - - spec: intel-oneapi-mpi@2021.15%oneapi@2025.1.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.1.1 - modules: - - mpi/2021.15 - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 - modules: - - mkl/2024.2 - - spec: intel-oneapi-mkl@2025.1%oneapi@2025.1.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.1.1 - modules: - - mkl/2025.1 - intel-oneapi-tbb: - externals: - - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 - modules: - - tbb/2021.13 - - spec: intel-oneapi-tbb@2022.1%oneapi@2025.1.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.1.1 - modules: - - tbb/2022.1 - intel-oneapi-runtime: - externals: - - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 - modules: - - tbb/2021.13 - - compiler-rt/2024.2.1 - - spec: intel-oneapi-runtime@2025.1.1%oneapi@2025.1.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.1.1 - modules: - - tbb/2022.1 - - compiler-rt/2025.1.1 From ff39465318e76cd31aba312c431abf6bab898edb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 29 Sep 2025 13:40:24 -0600 Subject: [PATCH 122/163] Update atlantis site config --- configs/sites/tier1/atlantis/packages.yaml | 21 +++----- .../tier1/atlantis/packages_gcc-13.4.0.yaml | 6 +-- .../atlantis/packages_oneapi-2024.2.1.yaml | 52 +++++-------------- 3 files changed, 22 insertions(+), 57 deletions(-) diff --git a/configs/sites/tier1/atlantis/packages.yaml b/configs/sites/tier1/atlantis/packages.yaml index 780638a13..0baa0fd9a 100644 --- a/configs/sites/tier1/atlantis/packages.yaml +++ b/configs/sites/tier1/atlantis/packages.yaml @@ -1,10 +1,7 @@ packages: # Modification of common packages - - ## Pin py-torch to 2.5.1 to avoid build errors with 2.6.0 - #py-torch: - # require: - # - '@2.5.1' + zlib-api: + buildable: False # All other packages listed alphabetically autoconf: @@ -74,11 +71,6 @@ packages: externals: - spec: openssh@8.0p1 prefix: /usr - # Can no longer use, issues with py-cryptography - #openssl: - # externals: - # - spec: openssl@1.1.1k - # prefix: /usr pkgconf: externals: - spec: pkgconf@1.4.2 @@ -103,7 +95,8 @@ packages: externals: - spec: wget@1.19.5 prefix: /usr - #zlib: - # externals: - # - spec: zlib@1.2.11 - # prefix: /usr + zlib: + buildable: False + externals: + - spec: zlib@1.2.11 + prefix: /usr diff --git a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml index 7a688d16f..3d1c814a0 100644 --- a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml +++ b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml @@ -2,7 +2,7 @@ packages: mpi: buildable: False require: - - "openmpi@4.1.8" + - openmpi@4.1.8 gcc: externals: - spec: gcc@13.4.0 languages:='c,c++,fortran' @@ -14,10 +14,6 @@ packages: c: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/gcc cxx: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/g++ fortran: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/gfortran - gcc-runtime: - externals: - - spec: gcc-runtime@13.4.0%gcc@13.4.0 - prefix: /gpfs/neptune/spack-stack/gcc-13.4.0 openmpi: externals: - spec: openmpi@4.1.8 diff --git a/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml index 352d4b687..143751e34 100644 --- a/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml +++ b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml @@ -1,17 +1,15 @@ packages: mpi: - buildable: False + buildable: false require: - - "intel-oneapi-mpi@2021.13" + - intel-oneapi-mpi@2021.13 intel-oneapi-compilers: + buildable: false externals: - spec: intel-oneapi-compilers@2024.2.1 prefix: /cm/shared/apps/intel/oneapi-2024.2.1 modules: - intel-oneapi-2024.2.1 - - tbb/2021.13 - - compiler-rt/2024.2.1 - - compiler/2024.2.1 extra_attributes: compilers: c: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/bin/icx @@ -24,10 +22,20 @@ packages: extra_rpaths: - /cm/local/apps/gcc/11.2.0/lib64 - /cm/local/apps/gcc/11.2.0/lib + gcc: + buildable: false + externals: + - spec: gcc@11.2.0 languages:='c,c++' + prefix: /cm/local/apps/gcc/11.2.0 + extra_attributes: + compilers: + c: /cm/local/apps/gcc/11.2.0/bin/gcc + cxx: /cm/local/apps/gcc/11.2.0/bin/g++ + fortran: /cm/local/apps/gcc/11.2.0/bin/gfortran intel-oneapi-mpi: buildable: false externals: - - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 + - spec: intel-oneapi-mpi@2021.13 prefix: /cm/shared/apps/intel/oneapi-2024.2.1 modules: - mpi/2021.13 @@ -37,35 +45,3 @@ packages: externals: - spec: intel-oneapi-mkl@2024.2 prefix: /cm/shared/apps/intel/oneapi-2024.2.1 - modules: - - mkl/2024.2 - intel-oneapi-tbb: - buildable: false - externals: - - spec: intel-oneapi-tbb@2021.13 - prefix: /cm/shared/apps/intel/oneapi-2024.2.1 - modules: - - tbb/2021.13 - intel-oneapi-runtime: - # LINK TO BUG REPORT HERE - #buildable: false - externals: - - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 - prefix: /cm/shared/apps/intel/oneapi-2024.2.1 - modules: - - intel-oneapi-2024.2.1 - - tbb/2021.13 - - compiler-rt/2024.2.1 - gcc: - externals: - - spec: gcc@11.2.0 languages:='c,c++' - prefix: /cm/local/apps/gcc/11.2.0 - extra_attributes: - compilers: - c: /cm/local/apps/gcc/11.2.0/bin/gcc - cxx: /cm/local/apps/gcc/11.2.0/bin/g++ - fortran: /cm/local/apps/gcc/11.2.0/bin/gfortran - gcc-runtime: - externals: - - spec: gcc-runtime@11.2.0%gcc@11.2.0 - prefix: /cm/local/apps/gcc/11.2.0 From 5fe51954ca41a728aa147a041c98839291aae193 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 29 Sep 2025 13:40:55 -0600 Subject: [PATCH 123/163] Use only external zlib instead of gcc-built zlib --- configs/common/packages_oneapi.yaml | 10 +++++----- configs/sites/tier2/blackpearl/packages.yaml | 15 +++++++-------- configs/sites/tier2/bounty/packages.yaml | 11 +++++++---- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index c890fd2fd..2c4af1fc6 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -50,8 +50,8 @@ packages: - '%gcc' # Prevent issues with fault Intel libraries # when using tar etc by building zlib-api with gcc - zlib: - require: - - '%gcc' - zlib-ng: - buildable: False + #zlib: + # require: + # - '%gcc' + #zlib-ng: + # buildable: False diff --git a/configs/sites/tier2/blackpearl/packages.yaml b/configs/sites/tier2/blackpearl/packages.yaml index 26e83dece..f45694ff4 100644 --- a/configs/sites/tier2/blackpearl/packages.yaml +++ b/configs/sites/tier2/blackpearl/packages.yaml @@ -1,9 +1,7 @@ packages: # Modification of common packages - # Pin py-torch to 2.5.1 to avoid build errors with 2.6.0 - #py-torch: - # require: - # - '@2.5.1' + zlib-api: + buildable: False # All other packages listed alphabetically autoconf: @@ -89,7 +87,8 @@ packages: externals: - spec: wget@1.21.1 prefix: /usr - #zlib: - # externals: - # - spec: zlib@1.2.11 - # prefix: /usr + zlib: + buildable: False + externals: + - spec: zlib@1.2.11 + prefix: /usr diff --git a/configs/sites/tier2/bounty/packages.yaml b/configs/sites/tier2/bounty/packages.yaml index b8206db38..35148d201 100644 --- a/configs/sites/tier2/bounty/packages.yaml +++ b/configs/sites/tier2/bounty/packages.yaml @@ -1,5 +1,7 @@ packages: # Modification of common packages + zlib-api: + buildable: False # All other packages listed alphabetically autoconf: @@ -103,7 +105,8 @@ packages: externals: - spec: wget@1.21.1 prefix: /usr - #zlib: - # externals: - # - spec: zlib@1.2.11 - # prefix: /usr + zlib: + buildable: False + externals: + - spec: zlib@1.2.11 + prefix: /usr From 97c32e2bb59f63e63c6b7135d888b5bdfa0ab7e2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 29 Sep 2025 13:43:27 -0600 Subject: [PATCH 124/163] Update submodule pointer for repos/builtin --- .gitmodules | 16 ++++++++-------- repos/builtin | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.gitmodules b/.gitmodules index 31ada4ea2..bb901f272 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,16 +1,16 @@ [submodule "spack"] path = spack - #url = https://github.com/jcsda/spack - ## Temporary, for transition to spack v1 code - ##branch = spack-stack-dev - #branch = feature/update_to_spack_v1 - url = https://github.com/climbfuji/spack - branch = feature/update_to_spack_v1p0p0 + url = https://github.com/jcsda/spack + # Temporary, for transition to spack v1 code + #branch = spack-stack-dev + branch = feature/update_to_spack_v1 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules branch = v1.3.0 [submodule "repos/builtin"] path = repos/builtin - url = https://github.com/jcsda/spack-packages - branch = spack-stack-dev + #url = https://github.com/jcsda/spack-packages + #branch = spack-stack-dev + url = https://github.com/climbfuji/spack-packages + branch = bugfix/oneapi2025p2_nasa_packages \ No newline at end of file diff --git a/repos/builtin b/repos/builtin index d974db373..a0799f831 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit d974db3739f2c83ace0f1459dbc3956feabc9e88 +Subproject commit a0799f83187613bc60c5b28cbd0fe8f9d305a865 From 0fcc6ace4c6025c2cb36d58a2d224df674110302 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 30 Sep 2025 14:22:22 +0000 Subject: [PATCH 125/163] Remove gcc-runtime constraint from configs/common/packages.yaml --- configs/common/packages.yaml | 3 --- repos/builtin | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 0d79d7d2e..a4da216bd 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -123,9 +123,6 @@ packages: g2tmpl: require: - '@1.13.0' - gcc-runtime: - require: - - '%gcc' # This version builds with Intel oneAPI compilers, newer/older versions don't gdal: require: diff --git a/repos/builtin b/repos/builtin index a0799f831..c752f04e0 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit a0799f83187613bc60c5b28cbd0fe8f9d305a865 +Subproject commit c752f04e03ab1a004c50a8a90a04f50cd07c98e0 From 17834b772ac0c1f3284550eeae51c9aa5daca22a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 30 Sep 2025 08:24:19 -0600 Subject: [PATCH 126/163] [skip ci] From 6ae756eb27d3b0f5d41f0e35e12181bced29f510 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 30 Sep 2025 08:42:04 -0600 Subject: [PATCH 127/163] [skip ci] Clean up NRL's and @climbfuji's configs, add external runtime libraries back in --- .../sites/tier1/atlantis/compilers.yaml.old | 113 ------------------ .../tier1/atlantis/packages_gcc-13.4.0.yaml | 7 ++ .../atlantis/packages_oneapi-2024.2.1.yaml | 14 +++ .../packages_oneapi.yaml.notyetupdated | 52 -------- .../blueback/compilers.yaml.nolongerused | 71 ----------- .../tier1/blueback/packages_gcc-13.3.0.yaml | 5 + .../blueback/packages_oneapi-2025.2.1.yaml | 19 ++- .../tier1/nautilus/packages_gcc-13.4.0.yaml | 9 ++ .../nautilus/packages_oneapi-2025.2.1.yaml | 18 ++- .../tier2/blackpearl/packages_gcc-13.2.1.yaml | 10 +- .../blackpearl/packages_oneapi-2024.2.1.yaml | 37 +++--- .../blackpearl/packages_oneapi-2025.1.0.yaml | 37 +++--- .../tier2/bounty/packages_clang-21.1.1.yaml | 1 + .../tier2/bounty/packages_gcc-13.3.1.yaml | 10 +- .../bounty/packages_oneapi-2025.1.1.yaml | 43 ------- .../bounty/packages_oneapi-2025.2.1.yaml | 36 +++--- .../spack-stack/stack/templates/mpi.lua | 4 - 17 files changed, 140 insertions(+), 346 deletions(-) delete mode 100644 configs/sites/tier1/atlantis/compilers.yaml.old delete mode 100644 configs/sites/tier1/atlantis/packages_oneapi.yaml.notyetupdated delete mode 100644 configs/sites/tier1/blueback/compilers.yaml.nolongerused delete mode 100644 configs/sites/tier2/bounty/packages_oneapi-2025.1.1.yaml diff --git a/configs/sites/tier1/atlantis/compilers.yaml.old b/configs/sites/tier1/atlantis/compilers.yaml.old deleted file mode 100644 index 849316449..000000000 --- a/configs/sites/tier1/atlantis/compilers.yaml.old +++ /dev/null @@ -1,113 +0,0 @@ -compilers: -- compiler: - spec: intel@2021.6.0 - paths: - cc: /cm/shared/apps/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icc - cxx: /cm/shared/apps/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icpc - f77: /cm/shared/apps/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort - fc: /cm/shared/apps/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - intel-oneapi - - compiler/2022.1.0 - environment: - prepend_path: - PATH: /cm/local/apps/gcc/11.2.0/bin - CPATH: /cm/local/apps/gcc/11.2.0/include - LD_LIBRARY_PATH: /cm/shared/apps/intel/oneapi/compiler/2022.1.0/linux/compiler/lib/intel64_lin:/cm/local/apps/gcc/11.2.0/lib64:/cm/local/apps/gcc/11.2.0/lib - extra_rpaths: [] -- compiler: - spec: gcc@11.2.0 - paths: - cc: /cm/local/apps/gcc/11.2.0/bin/gcc - cxx: /cm/local/apps/gcc/11.2.0/bin/g++ - f77: /cm/local/apps/gcc/11.2.0/bin/gfortran - fc: /cm/local/apps/gcc/11.2.0/bin/gfortran - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - gcc/11.2.0 - environment: {} - # For compiling parallel-netcdf; Bright must have removed some symbolic links required for OpenMPI 4.1.5a1 ... - extra_rpaths: [/gpfs/neptune/spack-stack/openmpi-4.1.5-fix-lib] -- compiler: - spec: gcc@13.4.0 - paths: - cc: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/gcc - cxx: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/g++ - f77: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/gfortran - fc: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/gfortran - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - gcc/13.4.0 - environment: - prepend_path: - MODULEPATH: /gpfs/neptune/spack-stack/openmpi-5.0.6/gcc-13.4.0/modulefiles:/gpfs/neptune/spack-stack/gcc-13.4.0/modulefiles - # For compiling parallel-netcdf; Bright must have removed some symbolic links required for OpenMPI 4.1.5a1 ... - extra_rpaths: [/gpfs/neptune/spack-stack/openmpi-4.1.5-fix-lib] -- compiler: - spec: oneapi@2024.2.1 - paths: - cc: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/bin/icx - cxx: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/bin/icpx - f77: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/bin/ifort - fc: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/bin/ifort - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - intel-oneapi-2024.2.1 - - tbb/2021.13 - - compiler-rt/2024.2.1 - - compiler/2024.2.1 - environment: - prepend_path: - PATH: /cm/local/apps/gcc/11.2.0/bin - CPATH: /cm/local/apps/gcc/11.2.0/include - LD_LIBRARY_PATH: /cm/local/apps/gcc/11.2.0/lib64:/cm/local/apps/gcc/11.2.0/lib - extra_rpaths: [] -- compiler: - spec: oneapi@2025.0.3 - paths: - cc: /gpfs/neptune/spack-stack/oneapi-2025.0.3/compiler/2025.0/bin/icx - cxx: /gpfs/neptune/spack-stack/oneapi-2025.0.3/compiler/2025.0/bin/icpx - f77: /gpfs/neptune/spack-stack/oneapi-2025.0.3/compiler/2025.0/bin/ifx - fc: /gpfs/neptune/spack-stack/oneapi-2025.0.3/compiler/2025.0/bin/ifx - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - umf/0.9.1 - - compiler-rt/2025.0.1 - - tbb/2022.0 - - compiler-intel-llvm/2025.0.1 - environment: - prepend_path: - PATH: /cm/local/apps/gcc/11.2.0/bin - CPATH: /cm/local/apps/gcc/11.2.0/include - LD_LIBRARY_PATH: /cm/local/apps/gcc/11.2.0/lib64:/cm/local/apps/gcc/11.2.0/lib - extra_rpaths: [] -- compiler: - spec: clang@20.1.5 - paths: - cc: /gpfs/neptune/spack-stack/llvm-20.1.5/bin/clang - cxx: /gpfs/neptune/spack-stack/llvm-20.1.5/bin/clang++ - f77: /gpfs/neptune/spack-stack/llvm-20.1.5/bin/flang-new - fc: /gpfs/neptune/spack-stack/llvm-20.1.5/bin/flang-new - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - llvm/20.1.5 - environment: - append_path: - LD_LIBRARY_PATH: /gpfs/neptune/spack-stack/llvm-20.1.5/lib:/gpfs/neptune/spack-stack/llvm-20.1.5/lib/x86_64-unknown-linux-gnu - prepend_path: - MODULEPATH: /gpfs/neptune/spack-stack/openmpi-5.0.6/llvm-20.1.5/modulefiles:/gpfs/neptune/spack-stack/llvm-20.1.5/modulefiles - extra_rpaths: [/gpfs/neptune/spack-stack/llvm-20.1.5/lib, /gpfs/neptune/spack-stack/llvm-20.1.5/lib/x86_64-unknown-linux-gnu ] - diff --git a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml index 3d1c814a0..1de501a42 100644 --- a/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml +++ b/configs/sites/tier1/atlantis/packages_gcc-13.4.0.yaml @@ -3,7 +3,13 @@ packages: buildable: False require: - openmpi@4.1.8 + gcc-runtime: + buildable: False + externals: + - spec: gcc-runtime@13.4.0%gcc@13.4.0 + prefix: /gpfs/neptune/spack-stack/gcc-13.4.0 gcc: + buildable: False externals: - spec: gcc@13.4.0 languages:='c,c++,fortran' prefix: /gpfs/neptune/spack-stack/gcc-13.4.0 @@ -15,6 +21,7 @@ packages: cxx: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/g++ fortran: /gpfs/neptune/spack-stack/gcc-13.4.0/bin/gfortran openmpi: + buildable: False externals: - spec: openmpi@4.1.8 prefix: /gpfs/neptune/spack-stack/openmpi-4.1.8/gcc-13.4.0 diff --git a/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml index 143751e34..2f2e111b1 100644 --- a/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml +++ b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml @@ -3,6 +3,15 @@ packages: buildable: false require: - intel-oneapi-mpi@2021.13 + intel-oneapi-runtime: + buildable: false + externals: + - spec: intel-oneapi-runtime@2024.2.1 + prefix: /cm/shared/apps/intel/oneapi-2024.2.1 + modules: + - intel-oneapi-2024.2.1 + - tbb/2021.13 + - compiler-rt/2024.2.1 intel-oneapi-compilers: buildable: false externals: @@ -22,6 +31,11 @@ packages: extra_rpaths: - /cm/local/apps/gcc/11.2.0/lib64 - /cm/local/apps/gcc/11.2.0/lib + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtime@11.2.0 + prefix: /cm/local/apps/gcc/11.2.0 gcc: buildable: false externals: diff --git a/configs/sites/tier1/atlantis/packages_oneapi.yaml.notyetupdated b/configs/sites/tier1/atlantis/packages_oneapi.yaml.notyetupdated deleted file mode 100644 index f9455f800..000000000 --- a/configs/sites/tier1/atlantis/packages_oneapi.yaml.notyetupdated +++ /dev/null @@ -1,52 +0,0 @@ -packages: - all: - compiler:: [oneapi@2024.2.1, oneapi@2025.0.3, gcc@11.2.0] - providers: - mpi:: [intel-oneapi-mpi@2021.13, intel-oneapi-mpi@2021.14] - mpi: - buildable: False - intel-oneapi-mpi: - externals: - - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 - prefix: /cm/shared/apps/intel/oneapi-2024.2.1 - modules: - - mpi/2021.13 - - slurm - - spec: intel-oneapi-mpi@2021.14%oneapi@2025.0.3 - prefix: /gpfs/neptune/spack-stack/oneapi-2025.0.3 - modules: - - mpi/2021.14 - - slurm - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2024.2 - prefix: /cm/shared/apps/intel/oneapi-2024.2.1 - modules: - - mkl/2024.2 - - spec: intel-oneapi-mkl@2025.0 - prefix: /gpfs/neptune/spack-stack/oneapi-2025.0.3 - modules: - - mkl/2025.0 - intel-oneapi-tbb: - externals: - - spec: intel-oneapi-tbb@2021.13 - prefix: /cm/shared/apps/intel/oneapi-2024.2.1 - modules: - - tbb/2021.13 - - spec: intel-oneapi-tbb@2022.0 - prefix: /gpfs/neptune/spack-stack/oneapi-2025.0.3 - modules: - - tbb/2022.0 - intel-oneapi-runtime: - externals: - - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 - prefix: /cm/shared/apps/intel/oneapi-2024.2.1 - modules: - - intel-oneapi-2024.2.1 - - tbb/2021.13 - - compiler-rt/2024.2.1 - - spec: intel-oneapi-runtime@2025.0.3%oneapi@2025.0.3 - prefix: /gpfs/neptune/spack-stack/oneapi-2025.0.3 - modules: - - tbb/2022.0 - - compiler-rt/2025.0.1 diff --git a/configs/sites/tier1/blueback/compilers.yaml.nolongerused b/configs/sites/tier1/blueback/compilers.yaml.nolongerused deleted file mode 100644 index 829fea599..000000000 --- a/configs/sites/tier1/blueback/compilers.yaml.nolongerused +++ /dev/null @@ -1,71 +0,0 @@ -compilers:: -- compiler: - spec: gcc@=13.3.0 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - target: x86_64 - modules: - - PrgEnv-gnu/8.6.0 - - gcc-native/13.2 - - cray-libsci/25.03.0 - - libfabric/1.22.0 - environment: - prepend_path: - LD_LIBRARY_PATH: '/opt/cray/libfabric/1.22.0/lib64:/opt/cray/pe/libsci/25.03.0/GNU/13.2/x86_64/lib' - set: - CONFIG_SITE: '' - CRAY_CPU_TARGET: x86-genoa - extra_rpaths: [] -- compiler: - spec: oneapi@=2024.2.1 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - target: x86_64 - modules: - - PrgEnv-intel/8.6.0 - - intel-oneapi/2024.2 - - cray-libsci/25.03.0 - - libfabric/1.22.0 - environment: - prepend_path: - LD_LIBRARY_PATH: '/opt/cray/libfabric/1.22.0/lib64:/opt/cray/pe/libsci/25.03.0/INTEL/2025.0/x86_64/lib' - set: - CONFIG_SITE: '' - CRAY_CPU_TARGET: x86-genoa - # Force ifort not ifx - INTEL_COMPILER_TYPE: 'RECOMMENDED' - extra_rpaths: [] -- compiler: - spec: oneapi@=2025.0.4 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - target: x86_64 - modules: - - PrgEnv-intel/8.6.0 - - intel-oneapi/2025.0 - - cray-libsci/25.03.0 - - libfabric/1.22.0 - environment: - prepend_path: - # The Cray compiler module does not include llvm-ar and other tools in the path - PATH: '/p/app/intel/oneapi-combined/compiler/2025.0/bin/compiler' - LD_LIBRARY_PATH: '/opt/cray/libfabric/1.22.0/lib64:/opt/cray/pe/libsci/25.03.0/INTEL/2025.0/x86_64/lib' - set: - CONFIG_SITE: '' - CRAY_CPU_TARGET: x86-genoa - extra_rpaths: [] diff --git a/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml index 876c2b482..aa5dea077 100644 --- a/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml +++ b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml @@ -3,6 +3,11 @@ packages: buildable: false require: - cray-mpich@8.1.32 + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtime@13.3.0 + prefix: /usr gcc: buildable: false externals: diff --git a/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml b/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml index d11efa8ad..0877199c6 100644 --- a/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml +++ b/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml @@ -3,12 +3,21 @@ packages: buildable: false require: - cray-mpich@8.1.32 + intel-oneapi-runtime: + buildable: false + externals: + - spec: intel-oneapi-runtime@2025.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1 + modules: + - PrgEnv-intel/8.6.0 + - intel-oneapi/2025.2.1 intel-oneapi-compilers: buildable: false externals: - spec: intel-oneapi-compilers@2025.2.1 prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1 modules: + - PrgEnv-intel/8.6.0 - intel-oneapi/2025.2.1 extra_attributes: compilers: @@ -18,10 +27,18 @@ packages: environment: set: CONFIG_SITE: '' + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtiem@13.3.0 + prefix: /usr + modules: + - PrgEnv-gnu/8.6.0 + - gcc-native/13.2 gcc: buildable: false externals: - - spec: gcc@13.3.0 languages:='c,c++,fortran' + - spec: gcc@13.3.0 languages:='c,c++' prefix: /usr modules: - PrgEnv-gnu/8.6.0 diff --git a/configs/sites/tier1/nautilus/packages_gcc-13.4.0.yaml b/configs/sites/tier1/nautilus/packages_gcc-13.4.0.yaml index 150c7e20f..027eb0b54 100644 --- a/configs/sites/tier1/nautilus/packages_gcc-13.4.0.yaml +++ b/configs/sites/tier1/nautilus/packages_gcc-13.4.0.yaml @@ -3,7 +3,15 @@ packages: buildable: False require: - openmpi@5.0.1 + gcc-runtime: + buildable: False + externals: + - spec: gcc-runtime@13.4.0 + prefix: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0 + modules: + - gcc/13.4.0 gcc: + buildable: False externals: - spec: gcc@13.4.0 languages:='c,c++,fortran' prefix: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0 @@ -15,6 +23,7 @@ packages: cxx: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/bin/g++ fortran: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/bin/gfortran openmpi: + buildable: False externals: - spec: openmpi@5.0.1 prefix: /p/app/penguin/openmpi/5.0.1/gcc-8.5.0 diff --git a/configs/sites/tier1/nautilus/packages_oneapi-2025.2.1.yaml b/configs/sites/tier1/nautilus/packages_oneapi-2025.2.1.yaml index 6e7177c3d..f404253fd 100644 --- a/configs/sites/tier1/nautilus/packages_oneapi-2025.2.1.yaml +++ b/configs/sites/tier1/nautilus/packages_oneapi-2025.2.1.yaml @@ -3,6 +3,15 @@ packages: buildable: false require: - intel-oneapi-mpi@2021.16 + intel-oneapi-runtime: + buildable: false + externals: + - spec: intel-oneapi-runtime@2025.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.2.1 + modules: + - tbb/2022.2 + - umf/0.11.0 + - compiler-rt/2025.2.1 intel-oneapi-compilers: buildable: false externals: @@ -25,10 +34,17 @@ packages: extra_rpaths: - /opt/rh/gcc-toolset-11/root/usr/lib64 - /opt/rh/gcc-toolset-11/root/usr/lib + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtime@11.2.1 + prefix: /opt/rh/gcc-toolset-11/root/usr + modules: + - scl/gcc-toolset-11 gcc: buildable: false externals: - - spec: gcc@11.2.1 languages:='c,c++,fortran' + - spec: gcc@11.2.1 languages:='c,c++' prefix: /opt/rh/gcc-toolset-11/root/usr modules: - scl/gcc-toolset-11 diff --git a/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml b/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml index 05ea6c755..2e8392fa2 100644 --- a/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml +++ b/configs/sites/tier2/blackpearl/packages_gcc-13.2.1.yaml @@ -2,7 +2,13 @@ packages: mpi: require: - "openmpi@=5.0.8" + gcc-runtime: + buildable: False + externals: + - spec: gcc-runtime@13.2.1 + prefix: /opt/rh/gcc-toolset-13/root/usr gcc: + buildable: False externals: - spec: gcc@13.2.1 languages:='c,c++,fortran' prefix: /opt/rh/gcc-toolset-13/root/usr @@ -11,7 +17,3 @@ packages: c: /opt/rh/gcc-toolset-13/root/usr/bin/gcc cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ fortran: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran - gcc-runtime: - externals: - - spec: gcc-runtime@13.2.1%gcc@13.2.1 - prefix: /opt/rh/gcc-toolset-13/root/usr diff --git a/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml b/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml index e4e3fbc83..6adf1a7fa 100644 --- a/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml +++ b/configs/sites/tier2/blackpearl/packages_oneapi-2024.2.1.yaml @@ -2,7 +2,12 @@ packages: mpi: buildable: False require: - - "intel-oneapi-mpi@2021.13" + - intel-oneapi-mpi@2021.13 + intel-oneapi-runtime: + buildable: false + externals: + - spec: intel-oneapi-runtime@2024.2.1 + prefix: /opt/intel/oneapi intel-oneapi-compilers: externals: - spec: intel-oneapi-compilers@2024.2.1 @@ -12,23 +17,13 @@ packages: c: /opt/intel/oneapi/2024.2/bin/icx cxx: /opt/intel/oneapi/2024.2/bin/icpx fortran: /opt/intel/oneapi/2024.2/bin/ifort - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.13 - prefix: /opt/intel/oneapi - intel-oneapi-mkl: + gcc-runtime: buildable: false externals: - - spec: intel-oneapi-mkl@2024.2 - prefix: /opt/intel/oneapi - intel-oneapi-runtime: - # LINK TO BUG REPORT HERE - #buildable: false - externals: - - spec: intel-oneapi-runtime@2024.2.1 - prefix: /opt/intel/oneapi + - spec: gcc-runtime@11.4.1 + prefix: /usr gcc: + buildable: false externals: - spec: gcc@11.4.1 languages:='c,c++' prefix: /usr @@ -37,7 +32,13 @@ packages: c: /usr/bin/gcc cxx: /usr/bin/g++ fortran: /usr/bin/gfortran - gcc-runtime: + intel-oneapi-mpi: + buildable: false externals: - - spec: gcc-runtime@11.4.1%gcc@11.4.1 - prefix: /usr + - spec: intel-oneapi-mpi@2021.13 + prefix: /opt/intel/oneapi + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2 + prefix: /opt/intel/oneapi diff --git a/configs/sites/tier2/blackpearl/packages_oneapi-2025.1.0.yaml b/configs/sites/tier2/blackpearl/packages_oneapi-2025.1.0.yaml index 90445d17f..b692e2239 100644 --- a/configs/sites/tier2/blackpearl/packages_oneapi-2025.1.0.yaml +++ b/configs/sites/tier2/blackpearl/packages_oneapi-2025.1.0.yaml @@ -2,7 +2,12 @@ packages: mpi: buildable: False require: - - "intel-oneapi-mpi@2021.15" + - intel-oneapi-mpi@2021.15 + intel-oneapi-runtime: + buildable: false + externals: + - spec: intel-oneapi-runtime@2025.1.0 + prefix: /opt/intel/oneapi intel-oneapi-compilers: externals: - spec: intel-oneapi-compilers@2025.1.0 @@ -12,23 +17,13 @@ packages: c: /opt/intel/oneapi/2025.1/bin/icx cxx: /opt/intel/oneapi/2025.1/bin/icpx fortran: /opt/intel/oneapi/2025.1/bin/ifx - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.15 - prefix: /opt/intel/oneapi - intel-oneapi-mkl: + gcc-runtime: buildable: false externals: - - spec: intel-oneapi-mkl@2025.1 - prefix: /opt/intel/oneapi - intel-oneapi-runtime: - # LINK TO BUG REPORT HERE - #buildable: false - externals: - - spec: intel-oneapi-runtime@2025.1.0 - prefix: /opt/intel/oneapi + - spec: gcc-runtime@11.4.1 + prefix: /usr gcc: + buildable: false externals: - spec: gcc@11.4.1 languages:='c,c++' prefix: /usr @@ -37,7 +32,13 @@ packages: c: /usr/bin/gcc cxx: /usr/bin/g++ fortran: /usr/bin/gfortran - gcc-runtime: + intel-oneapi-mpi: + buildable: false externals: - - spec: gcc-runtime@11.4.1%gcc@11.4.1 - prefix: /usr + - spec: intel-oneapi-mpi@2021.15 + prefix: /opt/intel/oneapi + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.1 + prefix: /opt/intel/oneapi diff --git a/configs/sites/tier2/bounty/packages_clang-21.1.1.yaml b/configs/sites/tier2/bounty/packages_clang-21.1.1.yaml index fa181f9fe..567b19442 100644 --- a/configs/sites/tier2/bounty/packages_clang-21.1.1.yaml +++ b/configs/sites/tier2/bounty/packages_clang-21.1.1.yaml @@ -3,6 +3,7 @@ packages: require: - "openmpi@=5.0.8" llvm: + buildable: False externals: - spec: llvm@21.1.1 +flang ~libomptarget prefix: /home/dom/prod/llvm-21.1.1 diff --git a/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml b/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml index 514689ece..052062dda 100644 --- a/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml +++ b/configs/sites/tier2/bounty/packages_gcc-13.3.1.yaml @@ -2,7 +2,13 @@ packages: mpi: require: - "openmpi@=5.0.8" + gcc-runtime: + buildable: False + externals: + - spec: gcc-runtime@13.3.1 + prefix: /opt/rh/gcc-toolset-13/root/usr gcc: + buildable: False externals: - spec: gcc@13.3.1 languages:='c,c++,fortran' prefix: /opt/rh/gcc-toolset-13/root/usr @@ -11,7 +17,3 @@ packages: c: /opt/rh/gcc-toolset-13/root/usr/bin/gcc cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ fortran: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran - gcc-runtime: - externals: - - spec: gcc-runtime@13.3.1%gcc@13.3.1 - prefix: /opt/rh/gcc-toolset-13/root/usr diff --git a/configs/sites/tier2/bounty/packages_oneapi-2025.1.1.yaml b/configs/sites/tier2/bounty/packages_oneapi-2025.1.1.yaml deleted file mode 100644 index 1d70245aa..000000000 --- a/configs/sites/tier2/bounty/packages_oneapi-2025.1.1.yaml +++ /dev/null @@ -1,43 +0,0 @@ -packages: - mpi: - buildable: False - require: - - "intel-oneapi-mpi@2021.15" - intel-oneapi-compilers: - externals: - - spec: intel-oneapi-compilers@2025.1.1 - prefix: /opt/intel/oneapi - extra_attributes: - compilers: - c: /opt/intel/oneapi/2025.1/bin/icx - cxx: /opt/intel/oneapi/2025.1/bin/icpx - fortran: /opt/intel/oneapi/2025.1/bin/ifx - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.15 - prefix: /opt/intel/oneapi - intel-oneapi-mkl: - buildable: false - externals: - - spec: intel-oneapi-mkl@2025.1 - prefix: /opt/intel/oneapi - intel-oneapi-runtime: - # LINK TO BUG REPORT HERE - #buildable: false - externals: - - spec: intel-oneapi-runtime@2025.1.1 - prefix: /opt/intel/oneapi - gcc: - externals: - - spec: gcc@11.5.0 languages:='c,c++' - prefix: /usr - extra_attributes: - compilers: - c: /usr/bin/gcc - cxx: /usr/bin/g++ - fortran: /usr/bin/gfortran - gcc-runtime: - externals: - - spec: gcc-runtime@11.5.0%gcc@11.5.0 - prefix: /usr diff --git a/configs/sites/tier2/bounty/packages_oneapi-2025.2.1.yaml b/configs/sites/tier2/bounty/packages_oneapi-2025.2.1.yaml index e24d32a6c..d27fc4dec 100644 --- a/configs/sites/tier2/bounty/packages_oneapi-2025.2.1.yaml +++ b/configs/sites/tier2/bounty/packages_oneapi-2025.2.1.yaml @@ -3,7 +3,13 @@ packages: buildable: False require: - "intel-oneapi-mpi@2021.16" + intel-oneapi-runtime: + buildable: false + externals: + - spec: intel-oneapi-runtime@2025.2.1 + prefix: /opt/intel/oneapi intel-oneapi-compilers: + buildable: false externals: - spec: intel-oneapi-compilers@2025.2.1 prefix: /opt/intel/oneapi @@ -12,23 +18,13 @@ packages: c: /opt/intel/oneapi/2025.2/bin/icx cxx: /opt/intel/oneapi/2025.2/bin/icpx fortran: /opt/intel/oneapi/2025.2/bin/ifx - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.16 - prefix: /opt/intel/oneapi - intel-oneapi-mkl: + gcc-runtime: buildable: false externals: - - spec: intel-oneapi-mkl@2025.2 - prefix: /opt/intel/oneapi - intel-oneapi-runtime: - # LINK TO BUG REPORT HERE - #buildable: false - externals: - - spec: intel-oneapi-runtime@2025.2.1 %intel-oneapi-compilers@2025.2.1 - prefix: /opt/intel/oneapi + - spec: gcc-runtime@11.5.0 %gcc@11.5.0 + prefix: /usr gcc: + buildable: false externals: - spec: gcc@11.5.0 languages:='c,c++' prefix: /usr @@ -37,7 +33,13 @@ packages: c: /usr/bin/gcc cxx: /usr/bin/g++ fortran: /usr/bin/gfortran - gcc-runtime: + intel-oneapi-mpi: + buildable: false externals: - - spec: gcc-runtime@11.5.0 %gcc@11.5.0 - prefix: /usr + - spec: intel-oneapi-mpi@2021.16 + prefix: /opt/intel/oneapi + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.2 + prefix: /opt/intel/oneapi diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua index d0aea6e95..3f7ddcf3e 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua @@ -26,10 +26,6 @@ setenv("MPICC", "@MPICC@") setenv("MPICXX", "@MPICXX@") setenv("MPIF77", "@MPIF77@") setenv("MPIF90", "@MPIF90@") -setenv("MPI_CC", "@MPICC@") -setenv("MPI_CXX", "@MPICXX@") -setenv("MPI_F77", "@MPIF77@") -setenv("MPI_F90", "@MPIF90@") # underlying compilers for mpi distributions setenv("I_MPI_CC", "@CC@") From 96d392a911e001263ef54ee0ccd25732cdfbd45b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 30 Sep 2025 08:44:04 -0600 Subject: [PATCH 128/163] Update .github/workflows/ubuntu-ci-x86_64-oneapi*.cfg to use external intel-oneapi-runtime --- .github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg | 3 +-- .github/workflows/ubuntu-ci-x86_64-oneapi.cfg | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg index d1d15f9da..bdff45cee 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg @@ -12,8 +12,7 @@ cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx fortran: /opt/intel/oneapi/compiler/2024.2/bin/ifx intel-oneapi-runtime: - # LINK TO BUG REPORT HERE - #buildable: false + buildable: false externals: - spec: intel-oneapi-runtime@2024.2.0 prefix: /opt/intel/oneapi diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.cfg b/.github/workflows/ubuntu-ci-x86_64-oneapi.cfg index d3e986def..72d1f604e 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.cfg +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.cfg @@ -12,8 +12,7 @@ cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx fortran: /opt/intel/oneapi/compiler/2024.2/bin/ifort intel-oneapi-runtime: - # LINK TO BUG REPORT HERE - #buildable: false + buildable: false externals: - spec: intel-oneapi-runtime@2024.2.0 prefix: /opt/intel/oneapi From 6692dc749f78199ed775ec4f3aa5178932649b77 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 30 Sep 2025 09:18:33 -0600 Subject: [PATCH 129/163] [skip ci] Add configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml, configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml --> configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml.donotuse --- .../blueback/packages_oneapi-2025.0.4.yaml | 69 +++++++++++++++++++ ...=> packages_oneapi-2025.2.1.yaml.donotuse} | 2 +- 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml rename configs/sites/tier1/blueback/{packages_oneapi-2025.2.1.yaml => packages_oneapi-2025.2.1.yaml.donotuse} (98%) diff --git a/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml b/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml new file mode 100644 index 000000000..1cbb0ef57 --- /dev/null +++ b/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml @@ -0,0 +1,69 @@ +packages: + mpi: + buildable: false + require: + - cray-mpich@8.1.32 + intel-oneapi-runtime: + buildable: false + externals: + - spec: intel-oneapi-runtime@2025.0.4 + prefix: /p/app/intel/oneapi-combined + modules: + - PrgEnv-intel/8.6.0 + - intel-oneapi/2025.0 + intel-oneapi-compilers: + buildable: false + externals: + - spec: intel-oneapi-compilers@2025.0.4 + prefix: /p/app/intel/oneapi-combined + modules: + - PrgEnv-intel/8.6.0 + - intel-oneapi/2025.0 + extra_attributes: + compilers: + c: /p/app/intel/oneapi-combined/compiler/2025.0/bin/icx + cxx: /p/app/intel/oneapi-combined/compiler/2025.0/bin/icpx + fortran: /p/app/intel/oneapi-combined/compiler/2025.0/bin/ifx + environment: + set: + CONFIG_SITE: '' + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtime@13.3.0 + prefix: /usr + modules: + - PrgEnv-gnu/8.6.0 + - gcc-native/13.2 + gcc: + buildable: false + externals: + - spec: gcc@13.3.0 languages:='c,c++' + prefix: /usr + modules: + - PrgEnv-gnu/8.6.0 + - gcc-native/13.2 + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + environment: + set: + CONFIG_SITE: '' + PE_GCC_EXTERNAL: native + cray-mpich: + buildable: false + externals: + - spec: cray-mpich@8.1.32 + prefix: /opt/cray/pe/mpich/8.1.32/ofi/INTEL/2022.1 + modules: + - libfabric/1.22.0 + - craype-network-ofi + - cray-mpich/8.1.32 + - craype-network-ofi + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.0 + prefix: /p/app/intel/oneapi-combined diff --git a/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml b/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml.donotuse similarity index 98% rename from configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml rename to configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml.donotuse index 0877199c6..7a932cce1 100644 --- a/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml +++ b/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml.donotuse @@ -30,7 +30,7 @@ packages: gcc-runtime: buildable: false externals: - - spec: gcc-runtiem@13.3.0 + - spec: gcc-runtime@13.3.0 prefix: /usr modules: - PrgEnv-gnu/8.6.0 From bd89947876548a63ca648945709c3cea063231d0 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 2 Oct 2025 13:29:09 -0600 Subject: [PATCH 130/163] trigger ci From 0d908a4a48b96f5e959011ce00e5c09e569c053a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 2 Oct 2025 20:02:50 -0600 Subject: [PATCH 131/163] Update configs/sites/tier1/ursa/compilers.yaml and configs/sites/tier1/ursa/packages_oneapi.yaml for building unified environment on Ursa for spack-stack develop as of 2025/10/02 (#1787) --- configs/sites/tier1/ursa/compilers.yaml | 9 ++++---- configs/sites/tier1/ursa/packages_oneapi.yaml | 21 ++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/configs/sites/tier1/ursa/compilers.yaml b/configs/sites/tier1/ursa/compilers.yaml index 4d96c8d5f..0f86cb3e2 100644 --- a/configs/sites/tier1/ursa/compilers.yaml +++ b/configs/sites/tier1/ursa/compilers.yaml @@ -13,9 +13,7 @@ compilers: - intel-oneapi-compilers/2024.2.1 environment: prepend_path: - PATH: '/usr/bin' - LD_LIBRARY_PATH: '/usr/lib:/usr/lib64' - CPATH: '/usr/include/c++/11/x86_64-redhat-linux' + CPATH: /usr/include/c++/11/x86_64-redhat-linux extra_rpaths: [] - compiler: @@ -30,5 +28,8 @@ compilers: target: x86_64 modules: - gcc/12.4.0 - environment: {} + environment: + prepend_path: + CPATH: /apps/spack-2024-12/linux-rocky9-x86_64/gcc-11.4.1/gcc-12.4.0-dsgnou52lpn2tus6mohdmcw5mjqmqrhj/lib/gcc/x86_64-pc-linux-gnu/12.4.0/include extra_rpaths: [] + diff --git a/configs/sites/tier1/ursa/packages_oneapi.yaml b/configs/sites/tier1/ursa/packages_oneapi.yaml index 5b33097a3..52d6f1f22 100644 --- a/configs/sites/tier1/ursa/packages_oneapi.yaml +++ b/configs/sites/tier1/ursa/packages_oneapi.yaml @@ -32,14 +32,15 @@ packages: modules: - intel-oneapi-mkl/2024.2.1 prefix: /apps/spack-2024-12/linux-rocky9-x86_64/gcc-11.4.1/intel-oneapi-mkl-2024.2.1-srqwrbzwo2k7hxuuhlrxttburs5jvlat - intel-oneapi-runtime: - externals: - - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 - prefix: /apps/spack-2024-12/linux-rocky9-x86_64/gcc-11.4.1/intel-oneapi-compilers-2024.2.1-oqhstbmawnrsdw472p4pjsopj547o6xs/compiler/2024.2 - modules: - - compiler-rt/2024.2.1 - gcc-runtime: - externals: - - spec: gcc-runtime@12.4.0%gcc@12.4.0 - prefix: /usr +# Commented out for spack v0.23 to avoid duplicate packages +# intel-oneapi-runtime: +# externals: +# - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 +# prefix: /apps/spack-2024-12/linux-rocky9-x86_64/gcc-11.4.1/intel-oneapi-compilers-2024.2.1-oqhstbmawnrsdw472p4pjsopj547o6xs/compiler/2024.2 +# modules: +# - compiler-rt/2024.2.1 +# gcc-runtime: +# externals: +# - spec: gcc-runtime@12.4.0%gcc@12.4.0 +# prefix: /usr From e79f10b0da624b8f9abacb617b959f997b12d85c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 3 Oct 2025 06:03:37 -0600 Subject: [PATCH 132/163] Bug fix in spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py: insert missing 'bin' in MPI compiler wrapper paths --- .../spack-stack/stack/meta_modules.py | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 8a4281aa5..ade683615 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -563,27 +563,39 @@ def custom_sort_key(entry): logging.debug(" ... ... MPIROOT: {}".format(line)) # Compiler wrapper environment variables + # Note: This doesn't return the correct names, see + # https://github.com/JCSDA/spack-stack/pull/1782#discussion_r2401650644 + #substitutes["MPICC"] = mpi_provider.prefix.bin.mpicc + #substitutes["MPICXX"] = mpi_provider.prefix.bin.mpicxx + #substitutes["MPIF77"] = mpi_provider.prefix.bin.mpif77 + #substitutes["MPIF90"] = mpi_provider.prefix.bin.mpif90 + ## Special case when using intel-oneapi-compilers with ifort instead of ifx + #if mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers" and \ + # not "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: + # substitutes["MPIF77"] = substitutes["MPIF77"].replace("mpiifx", "mpiifort") + # substitutes["MPIF90"] = substitutes["MPIF90"].replace("mpiifx", "mpiifort") if mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers": - substitutes["MPICC"] = os.path.join(mpi_provider.prefix, "mpiicx") - substitutes["MPICXX"] = os.path.join(mpi_provider.prefix, "mpiicpx") + substitutes["MPICC"] = os.path.join(mpi_provider.prefix.bin, "mpiicx") + substitutes["MPICXX"] = os.path.join(mpi_provider.prefix.bin, "mpiicpx") if "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and not "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: - substitutes["MPIF77"] = os.path.join(mpi_provider.prefix, "mpiifx") - substitutes["MPIF90"] = os.path.join(mpi_provider.prefix, "mpiifx") + substitutes["MPIF77"] = os.path.join(mpi_provider.prefix.bin, "mpiifx") + substitutes["MPIF90"] = os.path.join(mpi_provider.prefix.bin, "mpiifx") elif not "ifx" in COMPILER_SUBSTITUTES_SAVE["FC"] and "ifort" in COMPILER_SUBSTITUTES_SAVE["FC"]: - substitutes["MPIF77"] = os.path.join(mpi_provider.prefix, "mpiifort") - substitutes["MPIF90"] = os.path.join(mpi_provider.prefix, "mpiifort") + substitutes["MPIF77"] = os.path.join(mpi_provider.prefix.bin, "mpiifort") + substitutes["MPIF90"] = os.path.join(mpi_provider.prefix.bin, "mpiifort") else: raise Exception(f"For {mpi_provider.name}, cannot determine MPI wrapper from FC={COMPILER_SUBSTITUTES_SAVE['FC']}") elif mpi_provider.name == "intel-oneapi-mpi" and compiler.name == "intel-oneapi-compilers-classic": - substitutes["MPICC"] = os.path.join(mpi_provider.prefix, "mpiicc") - substitutes["MPICXX"] = os.path.join(mpi_provider.prefix, "mpiicpc") - substitutes["MPIF77"] = os.path.join(mpi_provider.prefix, "mpiifort") - substitutes["MPIF90"] = os.path.join(mpi_provider.prefix, "mpiifort") + substitutes["MPICC"] = os.path.join(mpi_provider.prefix.bin, "mpiicc") + substitutes["MPICXX"] = os.path.join(mpi_provider.prefix.bin, "mpiicpc") + substitutes["MPIF77"] = os.path.join(mpi_provider.prefix.bin, "mpiifort") + substitutes["MPIF90"] = os.path.join(mpi_provider.prefix.bin, "mpiifort") else: - substitutes["MPICC"] = os.path.join(mpi_provider.prefix, "mpicc") - substitutes["MPICXX"] = os.path.join(mpi_provider.prefix, "mpic++") - substitutes["MPIF77"] = os.path.join(mpi_provider.prefix, "mpif77") - substitutes["MPIF90"] = os.path.join(mpi_provider.prefix, "mpif90") + substitutes["MPICC"] = os.path.join(mpi_provider.prefix.bin, "mpicc") + substitutes["MPICXX"] = os.path.join(mpi_provider.prefix.bin, "mpic++") + substitutes["MPIF77"] = os.path.join(mpi_provider.prefix.bin, "mpif77") + substitutes["MPIF90"] = os.path.join(mpi_provider.prefix.bin, "mpif90") + # Also set the direct compiler environment variables substitutes["CC"] = COMPILER_SUBSTITUTES_SAVE["CC"] substitutes["CXX"] = COMPILER_SUBSTITUTES_SAVE["CXX"] From 4d47ebe62ba2bdeea3cf8c42c5f7414be31117e0 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 3 Oct 2025 10:31:00 -0600 Subject: [PATCH 133/163] Update submodule pointer for repos/builtin --- .gitmodules | 6 ++---- repos/builtin | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index bb901f272..068339c16 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,7 +10,5 @@ branch = v1.3.0 [submodule "repos/builtin"] path = repos/builtin - #url = https://github.com/jcsda/spack-packages - #branch = spack-stack-dev - url = https://github.com/climbfuji/spack-packages - branch = bugfix/oneapi2025p2_nasa_packages \ No newline at end of file + url = https://github.com/jcsda/spack-packages + branch = spack-stack-dev diff --git a/repos/builtin b/repos/builtin index c752f04e0..95466b32a 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit c752f04e03ab1a004c50a8a90a04f50cd07c98e0 +Subproject commit 95466b32a7aa54e4102da299b29b2facbf6547f5 From e0b05025622a86877a60a3a798b074de7be73bb5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 6 Oct 2025 13:33:33 +0000 Subject: [PATCH 134/163] Update .gitmodules and submodule pointer for repos/builtin for code review and testing --- .gitmodules | 7 +++++-- .../{packages_gcc-13.4.0.yaml => packages_gcc-13.3.1.yaml} | 0 repos/builtin | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) rename configs/sites/tier1/nautilus/{packages_gcc-13.4.0.yaml => packages_gcc-13.3.1.yaml} (100%) diff --git a/.gitmodules b/.gitmodules index 068339c16..066e3a986 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,5 +10,8 @@ branch = v1.3.0 [submodule "repos/builtin"] path = repos/builtin - url = https://github.com/jcsda/spack-packages - branch = spack-stack-dev + #url = https://github.com/jcsda/spack-packages + #branch = spack-stack-dev + url = https://github.com/climbfuji/spack-packages + branch = bugfix/esmf_lmpi_cxx + diff --git a/configs/sites/tier1/nautilus/packages_gcc-13.4.0.yaml b/configs/sites/tier1/nautilus/packages_gcc-13.3.1.yaml similarity index 100% rename from configs/sites/tier1/nautilus/packages_gcc-13.4.0.yaml rename to configs/sites/tier1/nautilus/packages_gcc-13.3.1.yaml diff --git a/repos/builtin b/repos/builtin index 95466b32a..735039ecd 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 95466b32a7aa54e4102da299b29b2facbf6547f5 +Subproject commit 735039ecd4ec0b40f1fd09159427917415b857f1 From d100362cf4672c9c0314a5a658f5284efca7ddcd Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 6 Oct 2025 13:34:58 +0000 Subject: [PATCH 135/163] Update Nautilus GCC site config and documentation --- .../tier1/nautilus/packages_gcc-13.3.1.yaml | 26 +++++++++---------- doc/source/PreConfiguredSites.rst | 8 +----- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/configs/sites/tier1/nautilus/packages_gcc-13.3.1.yaml b/configs/sites/tier1/nautilus/packages_gcc-13.3.1.yaml index 027eb0b54..59220d811 100644 --- a/configs/sites/tier1/nautilus/packages_gcc-13.3.1.yaml +++ b/configs/sites/tier1/nautilus/packages_gcc-13.3.1.yaml @@ -2,31 +2,31 @@ packages: mpi: buildable: False require: - - openmpi@5.0.1 + - openmpi@4.1.8 gcc-runtime: buildable: False externals: - - spec: gcc-runtime@13.4.0 - prefix: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0 + - spec: gcc-runtime@13.3.1 + prefix: /opt/rh/gcc-toolset-13/root/usr modules: - - gcc/13.4.0 + - scl/gcc-toolset-13 gcc: buildable: False externals: - - spec: gcc@13.4.0 languages:='c,c++,fortran' - prefix: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0 + - spec: gcc@13.3.1 languages:='c,c++,fortran' + prefix: /opt/rh/gcc-toolset-13/root/usr modules: - - gcc/13.4.0 + - scl/gcc-toolset-13 extra_attributes: compilers: - c: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/bin/gcc - cxx: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/bin/g++ - fortran: /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/bin/gfortran + c: /opt/rh/gcc-toolset-13/root/usr/bin/gcc + cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ + fortran: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran openmpi: buildable: False externals: - - spec: openmpi@5.0.1 - prefix: /p/app/penguin/openmpi/5.0.1/gcc-8.5.0 + - spec: openmpi@4.1.8 + prefix: /p/app/penguin/openmpi/4.1.8/gcc-13.3.1 modules: - - penguin/openmpi/5.0.1/gcc-8.5.0 + - penguin/openmpi/4.1.8/gcc-13.3.1 - slurm diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 7f805df1c..5bac2c35b 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -168,19 +168,13 @@ The following is required for building new spack environments with any supported NAVY HPCMP Nautilus ------------------------------ -The following is required for building new spack environments with any supported compiler on this platform. For ``gcc@13.4.0``, see the additional instructions below. +The following is required for building new spack environments with any supported compiler on this platform. .. code-block:: console umask 0022 module purge -For ``gcc@13.4.0``, further run: - -.. code-block:: console - - module use /p/app/projects/NEPTUNE/spack-stack/gcc-13.4.0/modulefiles - .. _Preconfigured_Sites_Blueback: From 010ae827165b6dbe8c3e29f7f4ad157fd1981ba3 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 6 Oct 2025 13:49:24 +0000 Subject: [PATCH 136/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 735039ecd..6f9d995a8 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 735039ecd4ec0b40f1fd09159427917415b857f1 +Subproject commit 6f9d995a8db73e641b703f0a148ef70c873abaa2 From c5ae5809c21085ed812a50878cd2a055e47002a8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 6 Oct 2025 15:43:55 +0000 Subject: [PATCH 137/163] Revert .gitmodules and update submodule pointer for repos/builtin --- .gitmodules | 6 ++---- repos/builtin | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 066e3a986..00d053673 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,8 +10,6 @@ branch = v1.3.0 [submodule "repos/builtin"] path = repos/builtin - #url = https://github.com/jcsda/spack-packages - #branch = spack-stack-dev - url = https://github.com/climbfuji/spack-packages - branch = bugfix/esmf_lmpi_cxx + url = https://github.com/jcsda/spack-packages + branch = spack-stack-dev diff --git a/repos/builtin b/repos/builtin index 6f9d995a8..1be5a4d5a 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 6f9d995a8db73e641b703f0a148ef70c873abaa2 +Subproject commit 1be5a4d5a2d667a3013f8b760465bb0df404af96 From 3dc2e15bd3361b806c9a39f383ff351adefb3a3b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 7 Oct 2025 05:36:18 -0600 Subject: [PATCH 138/163] Remove comment in configs/common/packages_oneapi.yaml --- configs/common/packages_oneapi.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index 2c4af1fc6..ebbceb029 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -48,10 +48,3 @@ packages: qt: require: - '%gcc' - # Prevent issues with fault Intel libraries - # when using tar etc by building zlib-api with gcc - #zlib: - # require: - # - '%gcc' - #zlib-ng: - # buildable: False From d5ad23e87d6339dcde3f101d00cc7e004f58ebe2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 7 Oct 2025 15:21:11 -0600 Subject: [PATCH 139/163] Update submodule pointer for repos/builtin --- .gitmodules | 1 - repos/builtin | 2 +- repos/spack-stack-dev-ref | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) create mode 160000 repos/spack-stack-dev-ref diff --git a/.gitmodules b/.gitmodules index 00d053673..068339c16 100644 --- a/.gitmodules +++ b/.gitmodules @@ -12,4 +12,3 @@ path = repos/builtin url = https://github.com/jcsda/spack-packages branch = spack-stack-dev - diff --git a/repos/builtin b/repos/builtin index 1be5a4d5a..025b1c3a5 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 1be5a4d5a2d667a3013f8b760465bb0df404af96 +Subproject commit 025b1c3a5d2081196d97e05c3dc792ece8ad0a2f diff --git a/repos/spack-stack-dev-ref b/repos/spack-stack-dev-ref new file mode 160000 index 000000000..50857d8ea --- /dev/null +++ b/repos/spack-stack-dev-ref @@ -0,0 +1 @@ +Subproject commit 50857d8ea172bb39aac4296feb446020b4249f7d From df5e5baa61e05d9ac366208f3f1f1a613e6e32b9 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 7 Oct 2025 15:25:50 -0600 Subject: [PATCH 140/163] Remove repos/spack-stack-dev-ref --- repos/spack-stack-dev-ref | 1 - 1 file changed, 1 deletion(-) delete mode 160000 repos/spack-stack-dev-ref diff --git a/repos/spack-stack-dev-ref b/repos/spack-stack-dev-ref deleted file mode 160000 index 50857d8ea..000000000 --- a/repos/spack-stack-dev-ref +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 50857d8ea172bb39aac4296feb446020b4249f7d From 4e94cb7a40c75b1516b7003d636d58f9992d7d22 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 7 Oct 2025 16:09:42 -0600 Subject: [PATCH 141/163] [skip ci] Update Narwhal site config (untested), clean up Blueback site config --- .../tier1/blueback/compilers.gcc-direct.tmp | 23 ----- .../blueback/packages_oneapi-2025.0.4.yaml | 1 - .../tier1/narwhal/compilers.gcc-direct.tmp | 24 ------ configs/sites/tier1/narwhal/compilers.yaml | 85 ------------------- configs/sites/tier1/narwhal/packages.yaml | 33 ++++--- .../tier1/narwhal/packages_gcc-12.2.0.yaml | 48 +++++++++++ configs/sites/tier1/narwhal/packages_gcc.yaml | 23 ----- .../sites/tier1/narwhal/packages_intel.yaml | 17 ---- .../narwhal/packages_oneapi-2024.2.0.yaml | 75 ++++++++++++++++ .../sites/tier1/narwhal/packages_oneapi.yaml | 27 ------ 10 files changed, 138 insertions(+), 218 deletions(-) delete mode 100644 configs/sites/tier1/blueback/compilers.gcc-direct.tmp delete mode 100644 configs/sites/tier1/narwhal/compilers.gcc-direct.tmp delete mode 100644 configs/sites/tier1/narwhal/compilers.yaml create mode 100644 configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml delete mode 100644 configs/sites/tier1/narwhal/packages_gcc.yaml delete mode 100644 configs/sites/tier1/narwhal/packages_intel.yaml create mode 100644 configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml delete mode 100644 configs/sites/tier1/narwhal/packages_oneapi.yaml diff --git a/configs/sites/tier1/blueback/compilers.gcc-direct.tmp b/configs/sites/tier1/blueback/compilers.gcc-direct.tmp deleted file mode 100644 index e12b07acc..000000000 --- a/configs/sites/tier1/blueback/compilers.gcc-direct.tmp +++ /dev/null @@ -1,23 +0,0 @@ -compilers:: -- compiler: - spec: gcc@=13.3.0 - paths: - cc: /usr/bin/gcc - cxx: /usr/bin/g++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - flags: {} - operating_system: sles15 - target: x86_64 - modules: - - PrgEnv-gnu/8.5.0 - - gcc-native/13.2 - - cray-libsci/24.07.0 - - libfabric/1.20.1 - environment: - prepend_path: - LD_LIBRARY_PATH: '/opt/cray/libfabric/1.20.1/lib64:/opt/cray/pe/libsci/24.07.0/GNU/12.3/x86_64/lib' - set: - CONFIG_SITE: '' - CRAY_CPU_TARGET: x86-genoa - extra_rpaths: [] diff --git a/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml b/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml index 1cbb0ef57..c689f12be 100644 --- a/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml +++ b/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml @@ -61,7 +61,6 @@ packages: - libfabric/1.22.0 - craype-network-ofi - cray-mpich/8.1.32 - - craype-network-ofi intel-oneapi-mkl: buildable: false externals: diff --git a/configs/sites/tier1/narwhal/compilers.gcc-direct.tmp b/configs/sites/tier1/narwhal/compilers.gcc-direct.tmp deleted file mode 100644 index f5250e760..000000000 --- a/configs/sites/tier1/narwhal/compilers.gcc-direct.tmp +++ /dev/null @@ -1,24 +0,0 @@ -compilers:: -- compiler: - spec: gcc@12.2.0 - paths: - # For cylc-dev, can't use Cray wrappers (https://github.com/spack/spack/issues/48515) - cc: /opt/cray/pe/gcc/12.2.0/snos/bin/gcc - cxx: /opt/cray/pe/gcc/12.2.0/snos/bin/g++ - f77: /opt/cray/pe/gcc/12.2.0/snos/bin/gfortran - fc: /opt/cray/pe/gcc/12.2.0/snos/bin/gfortran - flags: {} - operating_system: sles15 - target: x86_64 - modules: - - PrgEnv-gnu/8.4.0 - - gcc/12.2.0 - - cray-libsci/23.05.1.4 - - libfabric/1.12.1.2.2.1 - environment: - prepend_path: - LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/GNU/10.3/x86_64/lib' - set: - CRAYPE_LINK_TYPE: 'dynamic' - extra_rpaths: - - /opt/cray/pe/libsci/23.05.1.4/GNU/10.3/x86_64/lib diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml deleted file mode 100644 index 9e8306df7..000000000 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ /dev/null @@ -1,85 +0,0 @@ -compilers:: -- compiler: - spec: oneapi@2024.2.0 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - target: x86_64 - modules: - - PrgEnv-intel/8.4.0 - - intel/2024.2 - - cray-libsci/23.05.1.4 - - libfabric/1.12.1.2.2.1 - environment: - prepend_path: - PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' - CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include' - LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64' - append_path: - CPATH: '/opt/intel/oneapi_2024.2.0.634/compiler/2024.2/opt/compiler/include/intel64' - set: - CRAYPE_LINK_TYPE: 'dynamic' - # Force ifort not ifx - INTEL_COMPILER_TYPE: 'RECOMMENDED' - extra_rpaths: - - /opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib -- compiler: - spec: gcc@10.3.0 - paths: - # Still need Cray wrappers for most environments - cc: cc - cxx: CC - f77: ftn - fc: ftn - # For cylc-dev, can't use Cray wrappers (https://github.com/spack/spack/issues/48515) - #cc: /opt/cray/pe/gcc/10.3.0/snos/bin/gcc - #cxx: /opt/cray/pe/gcc/10.3.0/snos/bin/g++ - #f77: /opt/cray/pe/gcc/10.3.0/snos/bin/gfortran - #fc: /opt/cray/pe/gcc/10.3.0/snos/bin/gfortran - flags: {} - operating_system: sles15 - target: x86_64 - modules: - - PrgEnv-gnu/8.4.0 - - gcc/10.3.0 - - cray-libsci/23.05.1.4 - - libfabric/1.12.1.2.2.1 - environment: - prepend_path: - LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/GNU/10.3/x86_64/lib' - set: - CRAYPE_LINK_TYPE: 'dynamic' - extra_rpaths: - - /opt/cray/pe/libsci/23.05.1.4/GNU/10.3/x86_64/lib -- compiler: - spec: gcc@12.2.0 - paths: - # Still need Cray wrappers for most environments - cc: cc - cxx: CC - f77: ftn - fc: ftn - # For cylc-dev, can't use Cray wrappers (https://github.com/spack/spack/issues/48515) - #cc: /opt/cray/pe/gcc/12.2.0/snos/bin/gcc - #cxx: /opt/cray/pe/gcc/12.2.0/snos/bin/g++ - #f77: /opt/cray/pe/gcc/12.2.0/snos/bin/gfortran - #fc: /opt/cray/pe/gcc/12.2.0/snos/bin/gfortran - flags: {} - operating_system: sles15 - target: x86_64 - modules: - - PrgEnv-gnu/8.4.0 - - gcc/12.2.0 - - cray-libsci/23.05.1.4 - - libfabric/1.12.1.2.2.1 - environment: - prepend_path: - LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/GNU/10.3/x86_64/lib' - set: - CRAYPE_LINK_TYPE: 'dynamic' - extra_rpaths: - - /opt/cray/pe/libsci/23.05.1.4/GNU/10.3/x86_64/lib diff --git a/configs/sites/tier1/narwhal/packages.yaml b/configs/sites/tier1/narwhal/packages.yaml index 5cd2804bf..c84da0b8f 100644 --- a/configs/sites/tier1/narwhal/packages.yaml +++ b/configs/sites/tier1/narwhal/packages.yaml @@ -1,18 +1,9 @@ packages: -# Modification of common packages - # Use system zlib instead of spack-built zlib-ng - all: - providers: - zlib-api:: [zlib] + # Modification of common packages zlib-api: buildable: False - zlib: - buildable: False - externals: - - spec: zlib@1.2.11 - prefix: /usr -# All other packages listed alphabetically + # All other packages listed alphabetically autoconf: externals: - spec: autoconf@2.69 @@ -75,13 +66,13 @@ packages: externals: - spec: m4@1.4.18 prefix: /usr - mysql: - buildable: False - externals: - - spec: mysql@8.0.31 - prefix: /p/app/projects/NEPTUNE/spack-stack/mysql-8.0.31 - modules: - - mysql/8.0.31 + #mysql: + # buildable: False + # externals: + # - spec: mysql@8.0.31 + # prefix: /p/app/projects/NEPTUNE/spack-stack/mysql-8.0.31 + # modules: + # - mysql/8.0.31 perl: externals: - spec: perl@5.26.1~cpanm+shared+threads @@ -121,3 +112,9 @@ packages: externals: - spec: wget@1.20.3 prefix: /usr + zlib: + buildable: False + externals: + - spec: zlib@1.2.11 + prefix: /usr + diff --git a/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml b/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml new file mode 100644 index 000000000..0ccd5646b --- /dev/null +++ b/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml @@ -0,0 +1,48 @@ +packages: + mpi: + buildable: false + require: + - cray-mpich@8.1.26 + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtime@12.2.0 + prefix: /opt/cray/pe/gcc/12.2.0/snos + modules: + - PrgEnv-gnu/8.4.0 + - gcc/12.2.0 + gcc: + buildable: false + externals: + - spec: gcc@12.2.0 languages:='c,c++,fortran' + prefix: /opt/cray/pe/gcc/12.2.0/snos + modules: + - PrgEnv-gnu/8.4.0 + - gcc/12.2.0 + extra_attributes: + compilers: + c: /opt/cray/pe/gcc/12.2.0/snos/bin/gcc + cxx: /opt/cray/pe/gcc/12.2.0/snos/bin/g++ + fortran: /opt/cray/pe/gcc/12.2.0/snos/bin/gfortran + environment: + set: + CONFIG_SITE: '' + cray-mpich: + buildable: false + externals: + - spec: cray-mpich@8.1.26 + prefix: /opt/cray/pe/mpich/8.1.26/ofi/gnu/9.1 + modules: + - libfabric/1.12.1.2.2.1 + - craype-network-ucx + - cray-mpich-ucx/8.1.26 + # Require node-js earlier than 21 to prevent compile + # errors with Cray GNU compiler 12.2.0 + #node-js: + # require: + # - '@:20' + # Turn off valgrind variant for py-torch because + # of linker issues with Cray GNU compiler 12.2.0 + #py-torch: + # require: + # - '~valgrind' diff --git a/configs/sites/tier1/narwhal/packages_gcc.yaml b/configs/sites/tier1/narwhal/packages_gcc.yaml deleted file mode 100644 index ad4693d15..000000000 --- a/configs/sites/tier1/narwhal/packages_gcc.yaml +++ /dev/null @@ -1,23 +0,0 @@ -packages: - all: - compiler:: [gcc@12.2.0] - providers: - mpi:: [cray-mpich@8.1.26] - mpi: - buildable: False - cray-mpich: - externals: - - spec: cray-mpich@8.1.26%gcc@12.2.0 ~wrappers - modules: - - cray-mpich-ucx/8.1.26 - - craype-network-ucx - # Require node-js earlier than 21 to prevent compile - # errors with Cray GNU compiler 12.2.0 - node-js: - require: - - '@:20' - # Turn off valgrind variant for py-torch because - # of linker issues with Cray GNU compiler 12.2.0 - py-torch: - require: - - '~valgrind' diff --git a/configs/sites/tier1/narwhal/packages_intel.yaml b/configs/sites/tier1/narwhal/packages_intel.yaml deleted file mode 100644 index d2d3ff3cc..000000000 --- a/configs/sites/tier1/narwhal/packages_intel.yaml +++ /dev/null @@ -1,17 +0,0 @@ -packages: - all: - compiler:: [intel@2021.10.0,gcc@10.3.0] - providers: - mpi:: [cray-mpich@8.1.26] - mpi: - buildable: False - cray-mpich: - externals: - - spec: cray-mpich@8.1.26%intel@2021.10.0 ~wrappers - modules: - - cray-mpich-ucx/8.1.26 - - craype-network-ucx - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2023.2.0 - prefix: /opt/intel/oneapi_2023.2.0.49397 diff --git a/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml b/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml new file mode 100644 index 000000000..5728bdcf0 --- /dev/null +++ b/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml @@ -0,0 +1,75 @@ +packages: + mpi: + buildable: false + require: + - cray-mpich@8.1.26 + intel-oneapi-runtime: + buildable: false + externals: + - spec: intel-oneapi-runtime@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 + modules: + - PrgEnv-intel/8.4.0 + - intel/2024.2 + intel-oneapi-compilers: + buildable: false + externals: + - spec: intel-oneapi-compilers@2025.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 + modules: + - PrgEnv-intel/8.4.0 + - intel-oneapi/2024.2 + extra_attributes: + compilers: + c: /opt/intel/oneapi_2024.2.0.634/compiler/2024.2/bin/icx + cxx: /opt/intel/oneapi_2024.2.0.634/compiler/2024.2/bin/icpx + fortran: /opt/intel/oneapi_2024.2.0.634/compiler/2024.2/bin/ifort + environment: + set: + CONFIG_SITE: '' + # Force ifort not ifx + INTEL_COMPILER_TYPE: 'RECOMMENDED' + prepend_path: + PATH: /opt/cray/pe/gcc/12.2.0/snos/bin + CPATH: /opt/cray/pe/gcc/12.2.0/snos/include + extra_rpaths: + - /opt/cray/pe/gcc/12.2.0/snos/lib64 + - /opt/cray/pe/gcc/12.2.0/snos/lib + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtime@12.2.0 + prefix: /opt/cray/pe/gcc/12.2.0/snos + modules: + - PrgEnv-gnu/8.4.0 + - gcc/12.2.0 + gcc: + buildable: false + externals: + - spec: gcc@12.2.0 languages:='c,c++' + prefix: /opt/cray/pe/gcc/12.2.0/snos + modules: + - PrgEnv-gnu/8.4.0 + - gcc/12.2.0 + extra_attributes: + compilers: + c: /opt/cray/pe/gcc/12.2.0/snos/bin/gcc + cxx: /opt/cray/pe/gcc/12.2.0/snos/bin/g++ + fortran: /opt/cray/pe/gcc/12.2.0/snos/bin/gfortran + environment: + set: + CONFIG_SITE: '' + cray-mpich: + buildable: false + externals: + - spec: cray-mpich@8.1.32 + prefix: /opt/cray/pe/mpich/8.1.26/ofi/gnu/9.1 + modules: + - libfabric/1.12.1.2.2.1 + - craype-network-ucx + - cray-mpich-ucx/8.1.26 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2 + prefix: /opt/intel/oneapi_2024.2.0.634 diff --git a/configs/sites/tier1/narwhal/packages_oneapi.yaml b/configs/sites/tier1/narwhal/packages_oneapi.yaml deleted file mode 100644 index 151061331..000000000 --- a/configs/sites/tier1/narwhal/packages_oneapi.yaml +++ /dev/null @@ -1,27 +0,0 @@ -packages: - all: - compiler:: [oneapi@2024.2.0, gcc@10.3.0] - providers: - mpi:: [cray-mpich@8.1.26] - mpi: - buildable: False - cray-mpich: - externals: - - spec: cray-mpich@8.1.26~wrappers%oneapi@2024.2.0 - modules: - - cray-mpich-ucx/8.1.26 - - craype-network-ucx - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0 - prefix: /opt/intel/oneapi_2024.2.0.634 - intel-oneapi-tbb: - externals: - - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.0 - prefix: /opt/intel/oneapi_2024.2.0.634 - modules: - - tbb/2021.13 - intel-oneapi-runtime: - externals: - - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0 - prefix: /opt/intel/oneapi_2024.2.0.634 From 351a3375f295d23e5a8c7546be4c99c7c143b522 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 8 Oct 2025 11:12:54 -0600 Subject: [PATCH 142/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 025b1c3a5..55263acb2 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 025b1c3a5d2081196d97e05c3dc792ece8ad0a2f +Subproject commit 55263acb26d3f017d28b4f5619f2ec09b058bb53 From 14afde66f9718ade1c39a78a00bc3d0639e93821 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 8 Oct 2025 15:35:47 -0600 Subject: [PATCH 143/163] Final updates of develop from release 1.9.3 (#1790) Final round of updates from release 1.9.3 (branch release/1.9.0) for develop. An attempt will be made to pull these updatesinto the new feature/update_to_spack_v1 branch for changes that are not related to site configs, and for site configs that haven't been updated yet to work with spack v1. --- configs/common/packages.yaml | 1 + configs/sites/tier1/hera/packages.yaml | 11 +- configs/sites/tier1/hera/packages_oneapi.yaml | 2 +- configs/sites/tier1/hercules/compilers.yaml | 4 + .../sites/tier1/hercules/packages_oneapi.yaml | 7 ++ configs/sites/tier1/nas/README.md | 118 +++++++++++++++--- configs/sites/tier1/nas/packages.yaml | 8 ++ configs/sites/tier1/noaa-aws/README.md | 24 ++-- configs/sites/tier1/noaa-azure/README.md | 24 ++-- configs/sites/tier1/noaa-gcloud/README.md | 23 ++-- configs/sites/tier1/orion/compilers.yaml | 4 + configs/sites/tier1/orion/packages.yaml | 4 + .../sites/tier1/orion/packages_oneapi.yaml | 4 + configs/sites/tier1/ursa/compilers.yaml | 1 - configs/sites/tier1/ursa/packages.yaml | 4 + spack | 2 +- 16 files changed, 170 insertions(+), 71 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index ea95d6562..0296adb32 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -324,6 +324,7 @@ packages: wgrib2: require: - '@3.6.0' + - '+ipolates +netcdf' wrf-io: require: '@1.2.0' zstd: diff --git a/configs/sites/tier1/hera/packages.yaml b/configs/sites/tier1/hera/packages.yaml index d12be7fba..c7808255a 100644 --- a/configs/sites/tier1/hera/packages.yaml +++ b/configs/sites/tier1/hera/packages.yaml @@ -1,7 +1,8 @@ packages: all: + # https://github.com/JCSDA/spack-stack/issues/1781 # To support hecflow01 - target: [haswell] + #target: [haswell] providers: zlib-api:: [zlib] zlib-api: @@ -110,10 +111,10 @@ packages: mysql: buildable: False externals: - - spec: mysql@8.0.31 - prefix: /scratch1/NCEPDEV/global/spack-stack/apps/mysql-8.0.31 + - spec: mysql@8.0.36 + prefix: /contrib/spack-stack/installs/mysql/8.0.36 modules: - - mysql/8.0.31 + - mysql/8.0.36 ncurses: externals: - spec: ncurses@5.9.20130511+termlib abi=5 @@ -129,7 +130,7 @@ packages: qt: externals: - spec: qt@5.12.6 - prefix: /scratch1/NCEPDEV/nems/role.epic/installs/qt-5.12.6/qtbase + prefix: /contrib/spack-stack/installs/qt-5.12.6/qtbase rsync: externals: - spec: rsync@3.1.2 diff --git a/configs/sites/tier1/hera/packages_oneapi.yaml b/configs/sites/tier1/hera/packages_oneapi.yaml index b103ce146..7bd73cab6 100644 --- a/configs/sites/tier1/hera/packages_oneapi.yaml +++ b/configs/sites/tier1/hera/packages_oneapi.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [oneapi@2024.2.1] + compiler:: [oneapi@2024.2.1, gcc@13.3.0] providers: mpi:: [intel-oneapi-mpi@2021.13] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/hercules/compilers.yaml b/configs/sites/tier1/hercules/compilers.yaml index ade7aa15d..efe9f2712 100644 --- a/configs/sites/tier1/hercules/compilers.yaml +++ b/configs/sites/tier1/hercules/compilers.yaml @@ -10,11 +10,15 @@ compilers: operating_system: rocky9 target: x86_64 modules: + - spack-managed-x86-64_v3 - intel-oneapi-compilers/2024.2.1 environment: set: # https://github.com/ufs-community/ufs-weather-model/issues/2015#issuecomment-1864438186 I_MPI_EXTRA_FILESYSTEM: 'ON' + # Override system module settings for FC and F77 (they're set to ifx) + F77: '/apps/spack-managed-x86_64_v3-v1.0/gcc-11.3.1/intel-oneapi-compilers-2024.2.1-podbez65l57ms4uba527kg7pomxk5y3m/compiler/2024.2/bin/ifort' + FC: '/apps/spack-managed-x86_64_v3-v1.0/gcc-11.3.1/intel-oneapi-compilers-2024.2.1-podbez65l57ms4uba527kg7pomxk5y3m/compiler/2024.2/bin/ifort' prepend_path: PATH: /usr/bin LD_LIBRARY_PATH: /usr/lib:/usr/lib64 diff --git a/configs/sites/tier1/hercules/packages_oneapi.yaml b/configs/sites/tier1/hercules/packages_oneapi.yaml index 8b21e8a89..6eed0cd0a 100644 --- a/configs/sites/tier1/hercules/packages_oneapi.yaml +++ b/configs/sites/tier1/hercules/packages_oneapi.yaml @@ -15,6 +15,8 @@ packages: - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 prefix: /apps/spack-managed-x86_64_v3-v1.0/oneapi-2024.2.1/intel-oneapi-mpi-2021.13.1-3pv63eugwmse2xpeglxib4dr2oeb42g2 modules: + - spack-managed-x86-64_v3 + - intel-oneapi-compilers/2024.2.1 - intel-oneapi-mpi/2021.13.1 intel-oneapi-mkl: @@ -23,6 +25,7 @@ packages: - spec: intel-oneapi-mkl@2024.2.1 prefix: /apps/spack-managed-x86_64_v3-v1.0/gcc-11.3.1/intel-oneapi-mkl-2024.2.1-aeiool3i5jj4newwifvkhow5almp67rt modules: + - spack-managed-x86-64_v3 - intel-oneapi-mkl/2024.2.1 intel-oneapi-runtime: @@ -30,6 +33,7 @@ packages: - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 prefix: /apps/spack-managed-x86_64_v3-v1.0/oneapi-2024.2.1/intel-oneapi-runtime-2024.2.1-hl5zgdjaldynq35dq3yotclfy2vblybx modules: + - spack-managed-x86-64_v3 - intel-oneapi-compilers/2024.2.1 - intel-oneapi-runtime/2024.2.1 @@ -49,3 +53,6 @@ packages: py-scipy: require: - '%gcc' + zlib-ng: + require: + - '~shared' diff --git a/configs/sites/tier1/nas/README.md b/configs/sites/tier1/nas/README.md index 4b552809f..b3f570619 100644 --- a/configs/sites/tier1/nas/README.md +++ b/configs/sites/tier1/nas/README.md @@ -1,49 +1,131 @@ # How to build spack-stack at NAS -## Generic +In the commands below some will be run on login nodes (with internet access) and some +on compute nodes as, at NAS, you aren't allowed more than 2 processes on a login node. +## Clone spack-stack + +``` +git clone --recursive https://github.com/JCSDA/spack-stack.git -b release/1.9.0 spack-stack-1.9.3 +``` + +## Grab interactive node + +Since NAS limits you to 2 processes on a login node, you'll need to grab an interactive node. For example: ``` -git clone --recursive https://github.com/JCSDA/spack-stack.git -b release/1.9.0 spack-stack-1.9.1 +qsub -I -V -X -l select=1:ncpus=128:mpiprocs=128:model=mil_ait -l walltime=12:00:00 -W group_list=s1873 -m b -N Interactive ``` +will get you a Milan node for 12 hours + +## Setup spack-stack on each node -## oneapi +We will start on a login node with internet access. This is mainly needed for the +`spack mirror create` command which downloads all the source code for the packages. ``` -cd spack-stack-1.9.1 +cd spack-stack-1.9.3 . setup.sh -spack stack create env --name ue-oneapi-2024.2.0 --template unified-dev --site nas --compiler oneapi -cd envs/ue-oneapi-2024.2.0 -spack env activate . -spack concretize 2>&1 | tee log.concretize -spack install --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install ``` -NOTE: You might need to run the `spack install` command multiple times because sometimes -it just fails. But then you run it more and more and it will eventually succeed. +## Create environments + +We create two different environments, one for oneAPI and one for GCC. The commands below +are used to create the environments. You only need to do this once. + +### oneAPI + +To create the oneAPI environment, do: ``` -spack module tcl refresh -y -spack stack setup-meta-modules -spack env deactivate +spack stack create env --name ue-oneapi-2024.2.0 --template unified-dev --site nas --compiler oneapi +cd envs/ue-oneapi-2024.2.0 ``` -## gcc +### GCC + +To create the GCC environment, do: ``` -cd spack-stack-1.9.1 -. setup.sh spack stack create env --name ue-gcc-12.3.0 --template unified-dev --site nas --compiler gcc cd envs/ue-gcc-12.3.0 +``` + +## Activate environment + +Now enter the spack environment you just created: + +``` spack env activate . +``` + +NOTE: You need to make sure you do this in *any* terminal where you want to do any commmand +below with this environment. + +## Concretize and create source cache + +``` spack concretize 2>&1 | tee log.concretize -spack install --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install +``` + +## Create source cache (LOGIN NODE ONLY) + +Because this step downloads all the source code for all packages and all versions, it +should be done on a login node with internet access. + +``` +spack mirror create -a -d /nobackup/gmao_SIteam/spack-stack/source-cache +``` + +NOTE: Make sure you are in an environment when you run that `spack mirror create` command. Otherwise, +you will download *EVERY* package and *EVERY* version in spack! + +## Install packages + +Our install process will actually have (at least) three steps. This is because of the `crtm` package +which requires internet access at build time. + +### Install crtm dependencies (COMPUTE NODE) + +``` +spack install -j 10 --verbose --fail-fast --show-log-on-error --no-check-signature --only dependencies crtm 2>&1 | tee log.install.crtm_dependencies +``` + +### Install crtm (LOGIN NODE) + +``` +spack install -j 2 --verbose --fail-fast --show-log-on-error --no-check-signature crtm 2>&1 | tee log.install.crtm +``` + +Note we are only using 2 processes here because NAS limits you to 2 processes on a login node. + +### Install rest of packages (COMPUTE NODE) + +``` +spack install -j 10 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install.after_crtm ``` NOTE: You might need to run the `spack install` command multiple times because sometimes it just fails. But then you run it more and more and it will eventually succeed. +### Packages needing internet access to build + +If you encounter other packages that need internet access to build, you can install them with: + +``` +spack install -j 2 --verbose --fail-fast --show-log-on-error --no-check-signature |& tee log.install. +``` + +Then, once that package is built, you can go back to the compute node and run the `spack install` command again. + +## Update module files and setup meta-modules + ``` spack module tcl refresh -y spack stack setup-meta-modules +``` + +## Deactivate environment + +``` spack env deactivate ``` diff --git a/configs/sites/tier1/nas/packages.yaml b/configs/sites/tier1/nas/packages.yaml index 0fde05ef8..7aed2e62e 100644 --- a/configs/sites/tier1/nas/packages.yaml +++ b/configs/sites/tier1/nas/packages.yaml @@ -40,10 +40,18 @@ packages: externals: - spec: gawk@4.2.1 prefix: /usr + gh: + externals: + - spec: gh@2.78.0 + prefix: /nobackup/gmao_SIteam/gh/2.78.0 git: externals: - spec: git@2.43.5~tcltk prefix: /usr + git-lfs: + externals: + - spec: git-lfs@3.7.0 + prefix: /nobackup/gmao_SIteam/git-lfs/3.7.0 gmake: externals: - spec: gmake@4.2.1 diff --git a/configs/sites/tier1/noaa-aws/README.md b/configs/sites/tier1/noaa-aws/README.md index 049683e43..bfa6938a4 100644 --- a/configs/sites/tier1/noaa-aws/README.md +++ b/configs/sites/tier1/noaa-aws/README.md @@ -1,6 +1,6 @@ # Provisiong ParallelWorks AWS clusters -## Steps to perform before installing spack-stack version 1.9.2 +## Steps to perform before installing spack-stack version 1.9.3 sudo su - chmod 777 /contrib @@ -8,30 +8,22 @@ yum install -y qt5-qtbase-devel yum install -y qt5-qtsvg-devel -## Steps to install spack-stack version 1.9.2 +## Steps to install spack-stack version 1.8.0 sudo su - chmod 777 /contrib module purge -module unuse /opt/cray/craype/default/modulefiles -module unuse /opt/cray/modulefiles -### For noaa-aws, run the line below as well: -module unuse /opt/intel/impi/2019.5.281/intel64/modulefiles -module load gnu -module load intel/2024.2.1 -module load impi/2024.2.1 -module unload gnu cd /contrib/spack-stack-rocky8/ -git clone --recursive https://github.com/JCSDA/spack-stack -b release/1.9.0 spack-stack-1.9.2 -cd spack-stack-1.9.2 +git clone --recursive https://github.com/JCSDA/spack-stack -b release/1.9.0 spack-stack-1.9.3 +cd spack-stack-1.9.3 . setup.sh spack stack create env --name ue-oneapi-2024.2.1 --template unified-dev --site noaa-aws --compiler oneapi cd envs/ue-oneapi-2024.2.1 spack env activate . spack concretize 2>&1 | tee log.concretize -spack install --verbose 2>&1 | tee log.install +spack install --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install spack module lmod refresh -y spack stack setup-meta-modules @@ -40,12 +32,12 @@ spack stack setup-meta-modules sudo su - chmod 777 /contrib -cd /contrib/spack-stack-rocky8/spack-stack-1.9.2 +cd /contrib/spack-stack-rocky8/spack-stack-1.9.3 . setup.sh -spack stack create env --name gsi-oneapi-2024.2.1 --template gsi-addon-dev --site noaa-aws --upstream /contrib/spack-stack-rocky8/spack-stack-1.9.2/envs/ue-intel-2024.2.1/install --compiler intel +spack stack create env --name gsi-oneapi-2024.2.1 --template gsi-addon-dev --site noaa-aws --upstream /contrib/spack-stack-rocky8/spack-stack-1.9.3/envs/ue-oneapi-2024.2.1/install --compiler oneapi cd envs/gsi-oneapi-2024.2.1 spack env activate . spack concretize 2>&1 | tee log.concretize -spack install --verbose 2>&1 | tee log.install +spack install --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install spack module lmod refresh --upstream-modules spack stack setup-meta-modules diff --git a/configs/sites/tier1/noaa-azure/README.md b/configs/sites/tier1/noaa-azure/README.md index b0be84729..3275b3ef2 100644 --- a/configs/sites/tier1/noaa-azure/README.md +++ b/configs/sites/tier1/noaa-azure/README.md @@ -1,6 +1,6 @@ # Provisiong ParallelWorks Azure clusters -## Steps to perform before installing spack-stack version 1.9.2 +## Steps to perform before installing spack-stack version 1.9.3 sudo su - chmod 777 /contrib @@ -8,28 +8,22 @@ yum install -y qt5-qtbase-devel yum install -y qt5-qtsvg-devel -## Steps to install spack-stack version 1.9.2 +## Steps to install spack-stack version 1.8.0 sudo su - chmod 777 /contrib module purge -module unuse /opt/cray/craype/default/modulefiles -module unuse /opt/cray/modulefiles -module load gnu -module load intel/2024.2.1 -module load impi/2024.2.1 -module unload gnu cd /contrib/spack-stack-rocky8/ -git clone --recursive https://github.com/JCSDA/spack-stack -b release/1.9.0 spack-stack-1.9.2 -cd spack-stack-1.9.2 +git clone --recursive https://github.com/JCSDA/spack-stack -b release/1.9.0 spack-stack-1.9.3 +cd spack-stack-1.9.3 . setup.sh spack stack create env --name ue-oneapi-2024.2.1 --template unified-dev --site noaa-azure --compiler oneapi -cd envs/ue-intel-2024.2.1 +cd envs/ue-oneapi-2024.2.1 spack env activate . spack concretize 2>&1 | tee log.concretize -spack install --verbose 2>&1 | tee log.install +spack install --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install spack module lmod refresh -y spack stack setup-meta-modules @@ -38,12 +32,12 @@ spack stack setup-meta-modules sudo su - chmod 777 /contrib -cd /contrib/spack-stack-rocky8/spack-stack-1.9.2 +cd /contrib/spack-stack-rocky8/spack-stack-1.9.3 . setup.sh -spack stack create env --name gsi-oneapi-2024.2.1 --template gsi-addon-dev --site noaa-azure --upstream /contrib/spack-stack-rocky8/spack-stack-1.9.2/envs/ue-oneapi-2024.2.1/install --compiler oneapi +spack stack create env --name gsi-oneapi-2024.2.1 --template gsi-addon-dev --site noaa-azure --upstream /contrib/spack-stack-rocky8/spack-stack-1.9.3/envs/ue-oneapi-2024.2.1/install --compiler oneapi cd envs/gsi-oneapi-2024.2.1 spack env activate . spack concretize 2>&1 | tee log.concretize -spack install --verbose 2>&1 | tee log.install +spack install --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install spack module lmod refresh --upstream-modules spack stack setup-meta-modules diff --git a/configs/sites/tier1/noaa-gcloud/README.md b/configs/sites/tier1/noaa-gcloud/README.md index b68681df2..dafdf0939 100644 --- a/configs/sites/tier1/noaa-gcloud/README.md +++ b/configs/sites/tier1/noaa-gcloud/README.md @@ -1,6 +1,6 @@ # Provisiong ParallelWorks GCP clusters -## Steps to perform before installing spack-stack version 1.9.2 +## Steps to perform before installing spack-stack version 1.9.3 sudo su - chmod 777 /contrib @@ -8,28 +8,22 @@ yum install -y qt5-qtbase-devel yum install -y qt5-qtsvg-devel -## Steps to install spack-stack version 1.9.2 +## Steps to install spack-stack version 1.8.0 sudo su - chmod 777 /contrib module purge -module unuse /opt/cray/craype/default/modulefiles -module unuse /opt/cray/modulefiles -module load gnu -module load intel/2024.2.1 -module load impi/2024.2.1 -module unload gnu cd /contrib/spack-stack-rocky8/ -git clone --recursive https://github.com/JCSDA/spack-stack -b release/1.9.0 spack-stack-1.9.2 -cd spack-stack-1.9.2 +git clone --recursive https://github.com/JCSDA/spack-stack -b release/1.9.0 spack-stack-1.9.3 +cd spack-stack-1.9.3 . setup.sh spack stack create env --name ue-oneapi-2024.2.1 --template unified-dev --site noaa-gcloud --compiler oneapi cd envs/ue-oneapi-2024.2.1 spack env activate . spack concretize 2>&1 | tee log.concretize -spack install --verbose 2>&1 | tee log.install +spack install --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install spack module lmod refresh -y spack stack setup-meta-modules @@ -38,11 +32,12 @@ spack stack setup-meta-modules sudo su - chmod 777 /contrib -cd /contrib/spack-stack-rocky8/spack-stack-1.9.2 +cd /contrib/spack-stack-rocky8/spack-stack-1.9.3 . setup.sh -spack stack create env --name gsi-oneapi-2024.2.1 --template gsi-addon-dev --site noaa-gcloud --upstream /contrib/spack-stack-rocky8/spack-stack-1.9.2/envs/ue-oneapi-2024.2.1/install --compiler oneapi +spack stack create env --name gsi-oneapi-2024.2.1 --template gsi-addon-dev --site noaa-gcloud --upstream /contrib/spack-stack-rocky8/spack-stack-1.9.3/envs/ue-oneapi-2024.2.1/install --compiler oneapi cd envs/gsi-oneapi-2024.2.1 +spack env activate . spack concretize 2>&1 | tee log.concretize -spack install --verbose 2>&1 | tee log.install +spack install --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install spack module lmod refresh --upstream-modules spack stack setup-meta-modules diff --git a/configs/sites/tier1/orion/compilers.yaml b/configs/sites/tier1/orion/compilers.yaml index 76486886f..22b211b65 100644 --- a/configs/sites/tier1/orion/compilers.yaml +++ b/configs/sites/tier1/orion/compilers.yaml @@ -41,11 +41,15 @@ compilers: operating_system: rocky9 target: x86_64 modules: + - spack-managed-x86-64_v3 - intel-oneapi-compilers/2024.2.1 environment: set: # https://github.com/ufs-community/ufs-weather-model/issues/2015#issuecomment-1864438186 I_MPI_EXTRA_FILESYSTEM: 'ON' + # override system module settings for FC and F77 (they're set to ifx) + F77: '/apps/spack-managed-x86_64_v3-v1.0/gcc-11.3.1/intel-oneapi-compilers-2024.2.1-podbez65l57ms4uba527kg7pomxk5y3m/compiler/2024.2/bin/ifort' + FC: '/apps/spack-managed-x86_64_v3-v1.0/gcc-11.3.1/intel-oneapi-compilers-2024.2.1-podbez65l57ms4uba527kg7pomxk5y3m/compiler/2024.2/bin/ifort' prepend_path: PATH: /usr/bin LD_LIBRARY_PATH: /usr/lib:/usr/lib64 diff --git a/configs/sites/tier1/orion/packages.yaml b/configs/sites/tier1/orion/packages.yaml index d469336de..a7ef7c899 100644 --- a/configs/sites/tier1/orion/packages.yaml +++ b/configs/sites/tier1/orion/packages.yaml @@ -91,6 +91,10 @@ packages: externals: - spec: openssh@8.7p1 prefix: /usr + openssl: + externals: + - spec: openssl@3.0.1 + prefix: /usr perl: externals: - spec: perl@5.32.1~cpanm+opcode+open+shared+threads diff --git a/configs/sites/tier1/orion/packages_oneapi.yaml b/configs/sites/tier1/orion/packages_oneapi.yaml index e05d59425..744618221 100644 --- a/configs/sites/tier1/orion/packages_oneapi.yaml +++ b/configs/sites/tier1/orion/packages_oneapi.yaml @@ -14,6 +14,8 @@ packages: externals: - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 modules: + - spack-managed-x86-64_v3 + - intel-oneapi-compilers/2024.2.1 - intel-oneapi-mpi/2021.13.1 prefix: /apps/spack-managed-x86_64_v3-v1.0/oneapi-2024.2.1/intel-oneapi-mpi-2021.13.1-3pv63eugwmse2xpeglxib4dr2oeb42g2 @@ -22,6 +24,7 @@ packages: externals: - spec: intel-oneapi-mkl@2024.2.1 modules: + - spack-managed-x86-64_v3 - intel-oneapi-mkl/2024.2.1 prefix: /apps/spack-managed-x86_64_v3-v1.0/gcc-11.3.1/intel-oneapi-mkl-2024.2.1-aeiool3i5jj4newwifvkhow5almp67rt @@ -30,6 +33,7 @@ packages: - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 prefix: /apps/spack-managed-x86_64_v3-v1.0/oneapi-2024.2.1/intel-oneapi-runtime-2024.2.1-hl5zgdjaldynq35dq3yotclfy2vblybx modules: + - spack-managed-x86-64_v3 - intel-oneapi-compilers/2024.2.1 - intel-oneapi-runtime/2024.2.1 diff --git a/configs/sites/tier1/ursa/compilers.yaml b/configs/sites/tier1/ursa/compilers.yaml index 0f86cb3e2..48a5b8cb4 100644 --- a/configs/sites/tier1/ursa/compilers.yaml +++ b/configs/sites/tier1/ursa/compilers.yaml @@ -32,4 +32,3 @@ compilers: prepend_path: CPATH: /apps/spack-2024-12/linux-rocky9-x86_64/gcc-11.4.1/gcc-12.4.0-dsgnou52lpn2tus6mohdmcw5mjqmqrhj/lib/gcc/x86_64-pc-linux-gnu/12.4.0/include extra_rpaths: [] - diff --git a/configs/sites/tier1/ursa/packages.yaml b/configs/sites/tier1/ursa/packages.yaml index 5a32c35c6..225a64617 100644 --- a/configs/sites/tier1/ursa/packages.yaml +++ b/configs/sites/tier1/ursa/packages.yaml @@ -143,6 +143,10 @@ packages: externals: - spec: sed@4.8 prefix: /usr + tar: + externals: + - spec: tar@1.34 + prefix: /usr texinfo: externals: - spec: texinfo@7.1 diff --git a/spack b/spack index b3fad4c1c..cce215c90 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit b3fad4c1c4803a83aa9cd2161003833ca721cf21 +Subproject commit cce215c90bc4502dc390df0f905517e75cd56732 From b446ba7630c44f53d296814712ec5ab42ecc553d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 8 Oct 2025 15:38:49 -0600 Subject: [PATCH 144/163] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 55cac7376..55f4ebba4 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 55cac73760a136318e2e98433f96d3a8b39efcf3 +Subproject commit 55f4ebba439ccc8ce473fa3e4e3c3ccf7f73b1cb From 0509e79e5a1f7f7e8d9065f086b0122833a48ad3 Mon Sep 17 00:00:00 2001 From: Rick Grubin Date: Thu, 9 Oct 2025 11:59:10 -0600 Subject: [PATCH 145/163] Update ursa packages.yaml to exclude system-provided tar (#1792) * Remove external tar; build into env instead --- configs/sites/tier1/ursa/packages.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/configs/sites/tier1/ursa/packages.yaml b/configs/sites/tier1/ursa/packages.yaml index 225a64617..5a32c35c6 100644 --- a/configs/sites/tier1/ursa/packages.yaml +++ b/configs/sites/tier1/ursa/packages.yaml @@ -143,10 +143,6 @@ packages: externals: - spec: sed@4.8 prefix: /usr - tar: - externals: - - spec: tar@1.34 - prefix: /usr texinfo: externals: - spec: texinfo@7.1 From eb93abc2af4cc3530b72071918158c7c37ddb68b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 9 Oct 2025 20:30:48 -0600 Subject: [PATCH 146/163] Hot fix for feature/update_to_spack_v1: update invalid syntax in mpi.lua --- spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua index 3f7ddcf3e..17ad7feb7 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua @@ -27,7 +27,7 @@ setenv("MPICXX", "@MPICXX@") setenv("MPIF77", "@MPIF77@") setenv("MPIF90", "@MPIF90@") -# underlying compilers for mpi distributions +-- underlying compilers for mpi distributions setenv("I_MPI_CC", "@CC@") setenv("I_MPI_CXX", "@CXX@") setenv("I_MPI_F77", "@F77@") From 107084da6e1a15ba7aa85650ea1ace080a022447 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 10 Oct 2025 05:43:42 -0600 Subject: [PATCH 147/163] Update .gitmodules and submodule pointer for repos/builtin --- .gitmodules | 6 ++++-- repos/builtin | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 068339c16..6f5b44a07 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,5 +10,7 @@ branch = v1.3.0 [submodule "repos/builtin"] path = repos/builtin - url = https://github.com/jcsda/spack-packages - branch = spack-stack-dev + #url = https://github.com/jcsda/spack-packages + #branch = spack-stack-dev + url = https://github.com/climbfuji/spack-packages + branch = feature/wgrib2_netcdf_spack_stack_dev diff --git a/repos/builtin b/repos/builtin index 55263acb2..5b62cf6f1 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 55263acb26d3f017d28b4f5619f2ec09b058bb53 +Subproject commit 5b62cf6f182ebc91b95b571de46f8b99c661f975 From fc3443ac19ec500a6b1da112e5e73e388dfd5077 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 10 Oct 2025 05:45:00 -0600 Subject: [PATCH 148/163] WIP: configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml --- .../tier1/narwhal/packages_oneapi-2024.2.0.yaml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml b/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml index 5728bdcf0..2fb10feda 100644 --- a/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml +++ b/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml @@ -14,7 +14,7 @@ packages: intel-oneapi-compilers: buildable: false externals: - - spec: intel-oneapi-compilers@2025.2.0 + - spec: intel-oneapi-compilers@2024.2.0 prefix: /opt/intel/oneapi_2024.2.0.634 modules: - PrgEnv-intel/8.4.0 @@ -33,6 +33,7 @@ packages: PATH: /opt/cray/pe/gcc/12.2.0/snos/bin CPATH: /opt/cray/pe/gcc/12.2.0/snos/include extra_rpaths: + - /opt/cray/libfabric/1.12.1.2.2.1/lib64 - /opt/cray/pe/gcc/12.2.0/snos/lib64 - /opt/cray/pe/gcc/12.2.0/snos/lib gcc-runtime: @@ -40,9 +41,9 @@ packages: externals: - spec: gcc-runtime@12.2.0 prefix: /opt/cray/pe/gcc/12.2.0/snos - modules: - - PrgEnv-gnu/8.4.0 - - gcc/12.2.0 + #modules: + #- PrgEnv-gnu/8.4.0 + #- gcc/12.2.0 gcc: buildable: false externals: @@ -62,12 +63,16 @@ packages: cray-mpich: buildable: false externals: - - spec: cray-mpich@8.1.32 - prefix: /opt/cray/pe/mpich/8.1.26/ofi/gnu/9.1 + - spec: cray-mpich@8.1.26 + prefix: /opt/cray/pe/mpich/8.1.26/ofi/intel/2022.1 modules: - libfabric/1.12.1.2.2.1 - craype-network-ucx - cray-mpich-ucx/8.1.26 + extra_attributes: + extra_rpaths: + - /opt/cray/libfabric/1.12.1.2.2.1/lib64 + # /opt/cray/pe/mpich/8.1.26/ofi/intel/2022.1/lib intel-oneapi-mkl: buildable: false externals: From d161f047def737adc2190ab5e50cfc8adc1bd142 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 10 Oct 2025 08:18:39 -0600 Subject: [PATCH 149/163] Add configs/sites/tier1/atlantis/packages_oneapi-2025.2.1.yaml --- .../atlantis/packages_oneapi-2025.2.1.yaml | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 configs/sites/tier1/atlantis/packages_oneapi-2025.2.1.yaml diff --git a/configs/sites/tier1/atlantis/packages_oneapi-2025.2.1.yaml b/configs/sites/tier1/atlantis/packages_oneapi-2025.2.1.yaml new file mode 100644 index 000000000..e127cfeb6 --- /dev/null +++ b/configs/sites/tier1/atlantis/packages_oneapi-2025.2.1.yaml @@ -0,0 +1,62 @@ +packages: + mpi: + buildable: false + require: + - intel-oneapi-mpi@2021.16 + intel-oneapi-runtime: + buildable: false + externals: + - spec: intel-oneapi-runtime@2025.2.1 + prefix: /gpfs/neptune/spack-stack/oneapi-2025.2.1 + intel-oneapi-compilers: + buildable: false + externals: + - spec: intel-oneapi-compilers@2025.2.1 + prefix: /gpfs/neptune/spack-stack/oneapi-2025.2.1 + modules: + - umf/0.11.0 + - tbb/2022.2 + - compiler-rt/2025.2.1 + - compiler/2025.2.1 + extra_attributes: + compilers: + c: /gpfs/neptune/spack-stack/oneapi-2025.2.1/compiler/2025.2/bin/icx + cxx: /gpfs/neptune/spack-stack/oneapi-2025.2.1/compiler/2025.2/bin/icpx + fortran: /gpfs/neptune/spack-stack/oneapi-2025.2.1/compiler/2025.2/bin/ifort + environment: + prepend_path: + PATH: /cm/local/apps/gcc/11.2.0/bin + CPATH: /cm/local/apps/gcc/11.2.0/include + extra_rpaths: + - /cm/local/apps/gcc/11.2.0/lib64 + - /cm/local/apps/gcc/11.2.0/lib + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtime@11.2.0 + prefix: /cm/local/apps/gcc/11.2.0 + gcc: + buildable: false + externals: + - spec: gcc@11.2.0 languages:='c,c++' + prefix: /cm/local/apps/gcc/11.2.0 + extra_attributes: + compilers: + c: /cm/local/apps/gcc/11.2.0/bin/gcc + cxx: /cm/local/apps/gcc/11.2.0/bin/g++ + fortran: /cm/local/apps/gcc/11.2.0/bin/gfortran + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.16 + prefix: /gpfs/neptune/spack-stack/oneapi-2025.2.1 + modules: + - mpi/2021.16 + - slurm + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.2 + prefix: /gpfs/neptune/spack-stack/oneapi-2025.2.1 + modules: + - mkl/2025.2 \ No newline at end of file From 3e4751da8153b40cea4d143d17af0086dbccd837 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 10 Oct 2025 08:27:46 -0600 Subject: [PATCH 150/163] Revert .gitmodules and update submodule pointer for repos/builtin --- .gitmodules | 6 ++---- repos/builtin | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 6f5b44a07..068339c16 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,7 +10,5 @@ branch = v1.3.0 [submodule "repos/builtin"] path = repos/builtin - #url = https://github.com/jcsda/spack-packages - #branch = spack-stack-dev - url = https://github.com/climbfuji/spack-packages - branch = feature/wgrib2_netcdf_spack_stack_dev + url = https://github.com/jcsda/spack-packages + branch = spack-stack-dev diff --git a/repos/builtin b/repos/builtin index 5b62cf6f1..c8fd82edf 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 5b62cf6f182ebc91b95b571de46f8b99c661f975 +Subproject commit c8fd82edff81d2ae6d5edf3348b4bf3409359f89 From a5612ca2ecf79aeba9be677796c5777d9eed2fde Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 10 Oct 2025 09:20:36 -0600 Subject: [PATCH 151/163] Bug fix in configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml --- configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml index 2f2e111b1..8a643cd53 100644 --- a/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml +++ b/configs/sites/tier1/atlantis/packages_oneapi-2024.2.1.yaml @@ -8,10 +8,6 @@ packages: externals: - spec: intel-oneapi-runtime@2024.2.1 prefix: /cm/shared/apps/intel/oneapi-2024.2.1 - modules: - - intel-oneapi-2024.2.1 - - tbb/2021.13 - - compiler-rt/2024.2.1 intel-oneapi-compilers: buildable: false externals: @@ -19,6 +15,9 @@ packages: prefix: /cm/shared/apps/intel/oneapi-2024.2.1 modules: - intel-oneapi-2024.2.1 + - tbb/2021.13 + - compiler-rt/2024.2.1 + - compiler/2024.2.1 extra_attributes: compilers: c: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/bin/icx From 05b055d4d28f876c7844aa8059b3117069b3c412 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 10 Oct 2025 13:11:30 -0600 Subject: [PATCH 152/163] Update .gitmodules and submodule pointer for repos/builtin --- .gitmodules | 6 ++++-- repos/builtin | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 068339c16..423d0cad3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,5 +10,7 @@ branch = v1.3.0 [submodule "repos/builtin"] path = repos/builtin - url = https://github.com/jcsda/spack-packages - branch = spack-stack-dev + #url = https://github.com/jcsda/spack-packages + #branch = spack-stack-dev + url = https://github.com/climbfuji/spack-packages + branch = bugfixes/kiwisolver_cartopy_ectrans diff --git a/repos/builtin b/repos/builtin index c8fd82edf..826c25f7f 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit c8fd82edff81d2ae6d5edf3348b4bf3409359f89 +Subproject commit 826c25f7f91e302b14d6225defe9c57a9f2f94c0 From 081903ff7e312f4edcead706bfafa5e1a631d92f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 10 Oct 2025 13:13:29 -0600 Subject: [PATCH 153/163] Bug fix in configs/sites/tier1/atlantis/packages_oneapi-2025.2.1.yaml: ifort -> ifx --- configs/sites/tier1/atlantis/packages_oneapi-2025.2.1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/sites/tier1/atlantis/packages_oneapi-2025.2.1.yaml b/configs/sites/tier1/atlantis/packages_oneapi-2025.2.1.yaml index e127cfeb6..d0ba656dc 100644 --- a/configs/sites/tier1/atlantis/packages_oneapi-2025.2.1.yaml +++ b/configs/sites/tier1/atlantis/packages_oneapi-2025.2.1.yaml @@ -22,7 +22,7 @@ packages: compilers: c: /gpfs/neptune/spack-stack/oneapi-2025.2.1/compiler/2025.2/bin/icx cxx: /gpfs/neptune/spack-stack/oneapi-2025.2.1/compiler/2025.2/bin/icpx - fortran: /gpfs/neptune/spack-stack/oneapi-2025.2.1/compiler/2025.2/bin/ifort + fortran: /gpfs/neptune/spack-stack/oneapi-2025.2.1/compiler/2025.2/bin/ifx environment: prepend_path: PATH: /cm/local/apps/gcc/11.2.0/bin From 083c8e2a80176ce64f5cc55354936a605021fa4d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 10 Oct 2025 13:13:47 -0600 Subject: [PATCH 154/163] Bug fixes in configs/sites/tier1/narwhal/packages_*.yaml --- configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml | 12 ++++++++---- .../tier1/narwhal/packages_oneapi-2024.2.0.yaml | 7 ++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml b/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml index 0ccd5646b..8d6d6c8f3 100644 --- a/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml +++ b/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml @@ -8,9 +8,6 @@ packages: externals: - spec: gcc-runtime@12.2.0 prefix: /opt/cray/pe/gcc/12.2.0/snos - modules: - - PrgEnv-gnu/8.4.0 - - gcc/12.2.0 gcc: buildable: false externals: @@ -27,15 +24,22 @@ packages: environment: set: CONFIG_SITE: '' + extra_rpaths: + - /opt/cray/libfabric/1.12.1.2.2.1/lib64 cray-mpich: buildable: false externals: - spec: cray-mpich@8.1.26 prefix: /opt/cray/pe/mpich/8.1.26/ofi/gnu/9.1 modules: - - libfabric/1.12.1.2.2.1 - craype-network-ucx - cray-mpich-ucx/8.1.26 + - libfabric/1.12.1.2.2.1 + extra_attributes: + extra_rpaths: + - /opt/cray/libfabric/1.12.1.2.2.1/lib64 + # - /opt/cray/pe/mpich/8.1.26/ofi/gnu/9.1/lib + # Require node-js earlier than 21 to prevent compile # errors with Cray GNU compiler 12.2.0 #node-js: diff --git a/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml b/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml index 2fb10feda..0333efa19 100644 --- a/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml +++ b/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml @@ -8,9 +8,6 @@ packages: externals: - spec: intel-oneapi-runtime@2024.2.0 prefix: /opt/intel/oneapi_2024.2.0.634 - modules: - - PrgEnv-intel/8.4.0 - - intel/2024.2 intel-oneapi-compilers: buildable: false externals: @@ -66,13 +63,13 @@ packages: - spec: cray-mpich@8.1.26 prefix: /opt/cray/pe/mpich/8.1.26/ofi/intel/2022.1 modules: - - libfabric/1.12.1.2.2.1 - craype-network-ucx - cray-mpich-ucx/8.1.26 + - libfabric/1.12.1.2.2.1 extra_attributes: extra_rpaths: - /opt/cray/libfabric/1.12.1.2.2.1/lib64 - # /opt/cray/pe/mpich/8.1.26/ofi/intel/2022.1/lib + # - /opt/cray/pe/mpich/8.1.26/ofi/intel/2022.1/lib intel-oneapi-mkl: buildable: false externals: From 28bc063252944cbbe83bab31f428588cd90f8fb4 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 20 Oct 2025 08:05:08 -0600 Subject: [PATCH 155/163] configs/common/modules_*: by default, do not create compiler modules --- configs/common/modules_lmod.yaml | 5 +++++ configs/common/modules_tcl.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 87b633b78..4b0d0370e 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -90,6 +90,11 @@ modules: - xtrans - xz - yaksa + # By default, do not create compiler modules + - apple-clang + - gcc + - intel-oneapi-compilers + - llvm # By default, do not create MPI modules - cray-mpich - intel-mpi diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 74186d8de..630c134c1 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -88,6 +88,11 @@ modules: - xtrans - xz - yaksa + # By default, do not create compiler modules + - apple-clang + - gcc + - intel-oneapi-compilers + - llvm # By default, do not create MPI modules - cray-mpich - intel-mpi From 3e9332c5c09c6e1851e96640705c049c2ee1d414 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 22 Oct 2025 06:35:10 -0600 Subject: [PATCH 156/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 826c25f7f..2a88b9d59 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 826c25f7f91e302b14d6225defe9c57a9f2f94c0 +Subproject commit 2a88b9d5968536e4d7a44a0c56d9065d33ef4fbb From e6a8a3b0c37792080b6697c1d07cc6ddadc10c48 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 22 Oct 2025 06:35:27 -0600 Subject: [PATCH 157/163] Add missing dependency on py-cfgrib to neptune-python-env --- .../spack_stack/packages/neptune_python_env/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py index 367b12446..f55671db4 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py @@ -25,6 +25,7 @@ class NeptunePythonEnv(BundlePackage): depends_on("esmf +python", type="run") depends_on("py-arch", type="run") + depends_on("py-cfgrib", type="run") depends_on("py-h5py", type="run") depends_on("py-netcdf4", type="run") depends_on("py-pandas", type="run") From 9e2b33b4e01bf4997fc3e40be19a80d6d47bb561 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 22 Oct 2025 06:35:45 -0600 Subject: [PATCH 158/163] Narwhal: bump libfabric to 1.22.0 --- configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml | 6 +++--- configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml b/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml index 8d6d6c8f3..a557c3407 100644 --- a/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml +++ b/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml @@ -25,7 +25,7 @@ packages: set: CONFIG_SITE: '' extra_rpaths: - - /opt/cray/libfabric/1.12.1.2.2.1/lib64 + - /opt/cray/libfabric/1.22.0/lib64 cray-mpich: buildable: false externals: @@ -34,10 +34,10 @@ packages: modules: - craype-network-ucx - cray-mpich-ucx/8.1.26 - - libfabric/1.12.1.2.2.1 + - libfabric/1.22.0 extra_attributes: extra_rpaths: - - /opt/cray/libfabric/1.12.1.2.2.1/lib64 + - /opt/cray/libfabric/1.22.0/lib64 # - /opt/cray/pe/mpich/8.1.26/ofi/gnu/9.1/lib # Require node-js earlier than 21 to prevent compile diff --git a/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml b/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml index 0333efa19..60ae19208 100644 --- a/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml +++ b/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml @@ -30,7 +30,7 @@ packages: PATH: /opt/cray/pe/gcc/12.2.0/snos/bin CPATH: /opt/cray/pe/gcc/12.2.0/snos/include extra_rpaths: - - /opt/cray/libfabric/1.12.1.2.2.1/lib64 + - /opt/cray/libfabric/1.22.0/lib64 - /opt/cray/pe/gcc/12.2.0/snos/lib64 - /opt/cray/pe/gcc/12.2.0/snos/lib gcc-runtime: @@ -65,10 +65,10 @@ packages: modules: - craype-network-ucx - cray-mpich-ucx/8.1.26 - - libfabric/1.12.1.2.2.1 + - libfabric/1.22.0 extra_attributes: extra_rpaths: - - /opt/cray/libfabric/1.12.1.2.2.1/lib64 + - /opt/cray/libfabric/1.22.0/lib64 # - /opt/cray/pe/mpich/8.1.26/ofi/intel/2022.1/lib intel-oneapi-mkl: buildable: false From f4e20f7faedb4d25ff24c7b223c8e6a728d23b0f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 22 Oct 2025 12:33:46 -0600 Subject: [PATCH 159/163] Add missing dependency py-cfgrib to neptune-python-env (#1796) --- .../repos/spack-stack/packages/neptune-python-env/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py index 97c494d3a..76098cdfb 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py @@ -24,6 +24,7 @@ class NeptunePythonEnv(BundlePackage): depends_on("esmf +python", type="run") depends_on("py-arch", type="run") + depends_on("py-cfgrib", type="run") depends_on("py-h5py", type="run") depends_on("py-netcdf4", type="run") depends_on("py-pandas", type="run") From 18b33f32e39c4bbc76f747d01d4da8689f3dde2b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 23 Oct 2025 16:04:22 -0600 Subject: [PATCH 160/163] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 2a88b9d59..991bda7df 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 2a88b9d5968536e4d7a44a0c56d9065d33ef4fbb +Subproject commit 991bda7df23859457a15e5c44bf56019c8c66b00 From 8f810021bc2bf08bb241cb0e9e290bf03db61c23 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 24 Oct 2025 07:38:16 -0600 Subject: [PATCH 161/163] For new site configs and CI testing, let spack create the compiler modules --- .../workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg | 18 ------------------ .github/workflows/ubuntu-ci-x86_64-oneapi.cfg | 18 ------------------ configs/sites/tier2/linux.default/modules.yaml | 15 ++++++++++++--- configs/sites/tier2/macos.default/modules.yaml | 15 ++++++++++++--- 4 files changed, 24 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg index bdff45cee..5650fdf74 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.cfg @@ -2,10 +2,6 @@ externals: - spec: intel-oneapi-compilers@2024.2.0 prefix: /opt/intel/oneapi - modules: - - tbb/2021.13 - - compiler-rt/2024.2.0 - - compiler/2024.2.0 extra_attributes: compilers: c: /opt/intel/oneapi/compiler/2024.2/bin/icx @@ -16,27 +12,13 @@ externals: - spec: intel-oneapi-runtime@2024.2.0 prefix: /opt/intel/oneapi - modules: - - tbb/2021.13 - - compiler-rt/2024.2.0 - intel-oneapi-tbb: - buildable: false - externals: - - spec: intel-oneapi-tbb@2021.13 - prefix: /opt/intel/oneapi - modules: - - tbb/2021.13 intel-oneapi-mkl: buildable: false externals: - spec: intel-oneapi-mkl@2024.2 prefix: /opt/intel/oneapi - modules: - - mkl/2024.2 intel-oneapi-mpi: buildable: false externals: - spec: intel-oneapi-mpi@2021.13 prefix: /opt/intel/oneapi - modules: - - mpi/2021.13 diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.cfg b/.github/workflows/ubuntu-ci-x86_64-oneapi.cfg index 72d1f604e..e5dc3782e 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.cfg +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.cfg @@ -2,10 +2,6 @@ externals: - spec: intel-oneapi-compilers@2024.2.0 prefix: /opt/intel/oneapi - modules: - - tbb/2021.13 - - compiler-rt/2024.2.0 - - compiler/2024.2.0 extra_attributes: compilers: c: /opt/intel/oneapi/compiler/2024.2/bin/icx @@ -16,27 +12,13 @@ externals: - spec: intel-oneapi-runtime@2024.2.0 prefix: /opt/intel/oneapi - modules: - - tbb/2021.13 - - compiler-rt/2024.2.0 - intel-oneapi-tbb: - buildable: false - externals: - - spec: intel-oneapi-tbb@2021.13 - prefix: /opt/intel/oneapi - modules: - - tbb/2021.13 intel-oneapi-mkl: buildable: false externals: - spec: intel-oneapi-mkl@2024.2 prefix: /opt/intel/oneapi - modules: - - mkl/2024.2 intel-oneapi-mpi: buildable: false externals: - spec: intel-oneapi-mpi@2021.13 prefix: /opt/intel/oneapi - modules: - - mpi/2021.13 diff --git a/configs/sites/tier2/linux.default/modules.yaml b/configs/sites/tier2/linux.default/modules.yaml index 57e6be9fe..30db6a396 100644 --- a/configs/sites/tier2/linux.default/modules.yaml +++ b/configs/sites/tier2/linux.default/modules.yaml @@ -4,7 +4,16 @@ modules: - tcl tcl: include: - # List of packages for which we need modules that are blacklisted by default - - openmpi + # List of packages for which we need modules that are blacklisted by default# By default, do not create compiler modules + # Compiler modules + - apple-clang + - gcc + - intel-oneapi-compilers + - llvm + # MPI modules + - cray-mpich + - intel-mpi + - intel-oneapi-mpi - mpich - - python + - mpt + - openmpi diff --git a/configs/sites/tier2/macos.default/modules.yaml b/configs/sites/tier2/macos.default/modules.yaml index 66048a90e..7845e1177 100644 --- a/configs/sites/tier2/macos.default/modules.yaml +++ b/configs/sites/tier2/macos.default/modules.yaml @@ -4,7 +4,16 @@ modules: - lmod lmod: include: - # List of packages for which we need modules that are blacklisted by default - - openmpi + # List of packages for which we need modules that are blacklisted by default# By default, do not create compiler modules + # Compiler modules + - apple-clang + - gcc + - intel-oneapi-compilers + - llvm + # MPI modules + - cray-mpich + - intel-mpi + - intel-oneapi-mpi - mpich - - python + - mpt + - openmpi From 289bc39667d1172991a5f1178b223432bba24745 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 24 Oct 2025 08:41:56 -0600 Subject: [PATCH 162/163] Blueback: workaround for cmake find mpi --- configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml | 4 ++++ configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml | 4 ++++ .../tier1/blueback/packages_oneapi-2025.2.1.yaml.donotuse | 5 ++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml index aa5dea077..582f42c0f 100644 --- a/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml +++ b/configs/sites/tier1/blueback/packages_gcc-13.3.0.yaml @@ -34,3 +34,7 @@ packages: - libfabric/1.22.0 - craype-network-ofi - cray-mpich/8.1.32 + extra_attributes: + environment: + prepend_path: + CMAKE_PREFIX_PATH: /p/app/projects/NEPTUNE/spack-stack/cmake-findmpi diff --git a/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml b/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml index c689f12be..136ed85d8 100644 --- a/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml +++ b/configs/sites/tier1/blueback/packages_oneapi-2025.0.4.yaml @@ -61,6 +61,10 @@ packages: - libfabric/1.22.0 - craype-network-ofi - cray-mpich/8.1.32 + extra_attributes: + environment: + prepend_path: + CMAKE_PREFIX_PATH: /p/app/projects/NEPTUNE/spack-stack/cmake-findmpi intel-oneapi-mkl: buildable: false externals: diff --git a/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml.donotuse b/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml.donotuse index 7a932cce1..f75d8ea2d 100644 --- a/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml.donotuse +++ b/configs/sites/tier1/blueback/packages_oneapi-2025.2.1.yaml.donotuse @@ -61,7 +61,10 @@ packages: - libfabric/1.22.0 - craype-network-ofi - cray-mpich/8.1.32 - - craype-network-ofi + extra_attributes: + environment: + prepend_path: + CMAKE_PREFIX_PATH: /p/app/projects/NEPTUNE/spack-stack/cmake-findmpi intel-oneapi-mkl: buildable: false externals: From 20229d036d7a39be6c1fbdf56ab3a2c3d874b3b9 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 24 Oct 2025 08:42:18 -0600 Subject: [PATCH 163/163] [skip ci] Narwhal: add extra rpath information for gcc compiler --- configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml | 2 ++ configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml b/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml index a557c3407..4da14962b 100644 --- a/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml +++ b/configs/sites/tier1/narwhal/packages_gcc-12.2.0.yaml @@ -26,6 +26,8 @@ packages: CONFIG_SITE: '' extra_rpaths: - /opt/cray/libfabric/1.22.0/lib64 + - /opt/cray/pe/gcc/12.2.0/snos/lib64 + - /opt/cray/pe/gcc/12.2.0/snos/lib cray-mpich: buildable: false externals: diff --git a/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml b/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml index 60ae19208..b3e6896f8 100644 --- a/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml +++ b/configs/sites/tier1/narwhal/packages_oneapi-2024.2.0.yaml @@ -38,9 +38,6 @@ packages: externals: - spec: gcc-runtime@12.2.0 prefix: /opt/cray/pe/gcc/12.2.0/snos - #modules: - #- PrgEnv-gnu/8.4.0 - #- gcc/12.2.0 gcc: buildable: false externals: @@ -57,6 +54,10 @@ packages: environment: set: CONFIG_SITE: '' + extra_rpaths: + - /opt/cray/libfabric/1.22.0/lib64 + - /opt/cray/pe/gcc/12.2.0/snos/lib64 + - /opt/cray/pe/gcc/12.2.0/snos/lib cray-mpich: buildable: false externals: