diff --git a/experiments/kripke/experiment.py b/experiments/kripke/experiment.py index 8ea0b97ad..5251645cb 100644 --- a/experiments/kripke/experiment.py +++ b/experiments/kripke/experiment.py @@ -36,16 +36,24 @@ class Kripke( ) variant( - "single_memory", + "gpu-aware-mpi", default=False, - description="Enable single memory space model in rocm", + values=(True, False), + description="Enable GPU-aware MPI", + ) + + variant( + "chai", + default=True, + values=(True, False), + description="Enable CHAI", ) variant( - "other", + "direct-device-plane", default=False, values=(True, False), - description="Set other input/environment variables", + description="Use direct device allocator in Umpire for plane fields", ) maintainers("pearce8") @@ -98,6 +106,7 @@ def compute_applications_section(self): if self.spec.satisfies("+throughput"): problem_spec = { "nzx": [ + 64, 80, 100, 120, @@ -106,8 +115,13 @@ def compute_applications_section(self): 180, 200, 220, + 240, + 260, + 280, + 300, ], "nzy": [ + 64, 80, 100, 120, @@ -116,8 +130,13 @@ def compute_applications_section(self): 180, 200, 220, + 240, + 260, + 280, + 300, ], "nzz": [ + 32, 40, 50, 60, @@ -126,11 +145,15 @@ def compute_applications_section(self): 90, 100, 110, + 120, + 130, + 140, + 150, ], "pool": 120, - "npx": [2, 2, 2, 2, 2, 2, 2, 2], - "npy": [2, 2, 2, 2, 2, 2, 2, 2], - "npz": [1, 1, 1, 1, 1, 1, 1, 1], + "npx": [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], + "npy": [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], + "npz": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], "ngroups": 48, "gs": 1, "nquad": 80, @@ -144,6 +167,34 @@ def compute_applications_section(self): "throughput_n": None, "throughput_p": None, } + elif self.spec.satisfies("+weak"): + problem_spec = { + "nzx": 80, + "nzy": 80, + "nzz": 40, + "pool": 120, + "npx": 2, + "npy": 2, + "npz": 1, + "ngroups": 48, + "gs": 1, + "nquad": 80, + "ds": 80, + "lorder": 4, + "layout": "GDZ", + "strong_n": None, + "strong_p": None, + "weak_n": lambda var, itr, dim, scaling_factor: var.val(dim) + * scaling_factor, + "weak_p": lambda var, itr, dim, scaling_factor: var.val(dim) + * scaling_factor, + "throughput_n": None, + "throughput_p": None, + } + problem_spec["nzx"] = 220 + problem_spec["nzy"] = 220 + problem_spec["nzz"] = 110 + problem_spec["pool"] = 105 else: problem_spec = { "nzx": 80, @@ -199,9 +250,6 @@ def compute_applications_section(self): self.add_experiment_variable("layout", problem_spec["layout"], True) self.add_experiment_variable("pool", problem_spec["pool"], True) - if self.spec.satisfies("+other"): - self.set_environment_variable("HSA_XNACK", 1) - # Set the variables required by the experiment self.set_required_variables( n_resources="{npx}*{npy}*{npz}", @@ -250,13 +298,25 @@ def compute_applications_section(self): else: self.add_experiment_variable("n_ranks", "{n_resources}", True) + if self.spec.satisfies("+gpu-aware-mpi"): + self.set_environment_variable("MPICH_GPU_IPC_CACHE_MAX_SIZE", 1000) + # self.set_environment_variable("MPICH_NOLOCAL", 1) + def compute_package_section(self): - # get package version - single_memory = ( - "+single_memory" - if self.spec.variants["single_memory"][0] - else "~single_memory" + gam = ( + "+gpu-aware-mpi" + if self.spec.variants["gpu-aware-mpi"][0] + else "~gpu-aware-mpi" + ) + chai = "+chai" if self.spec.variants["chai"][0] else "~chai" + direct_device_plane = ( + "+direct-device-plane" + if self.spec.variants["direct-device-plane"][0] + else "~direct-device-plane" ) self.add_package_spec( - self.name, [f"kripke{self.determine_version()} {single_memory} +mpi"] + self.name, + [ + f"kripke{self.determine_version()} {gam} {chai} {direct_device_plane} +mpi" + ], ) diff --git a/repos/spack_repo/benchpark/packages/chai/package.py b/repos/spack_repo/benchpark/packages/chai/package.py deleted file mode 100644 index 13ca1536b..000000000 --- a/repos/spack_repo/benchpark/packages/chai/package.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -from spack.package import * -from spack_repo.builtin.build_systems.cached_cmake import cmake_cache_option -from spack_repo.builtin.packages.chai.package import Chai as BuiltinChai - - -class Chai(BuiltinChai): - variant("single_memory", default=False, description="Enable single memory space model") - - conflicts("+single_memory", when="~rocm") - - version( - "2025.12.0", - tag="v2025.12.0", - commit="26d5646707e1848b0524379b12a7716e4a830a27", - submodules=False, - ) - - def initconfig_hardware_entries(self): - spec = self.spec - entries = super().initconfig_hardware_entries() - - if spec.satisfies("+single_memory"): - entries.append(cmake_cache_option("CHAI_THIN_GPU_ALLOCATE", True)) - entries.append(cmake_cache_option("CHAI_DISABLE_RM", True)) - - return entries diff --git a/repos/spack_repo/benchpark/packages/kripke/package.py b/repos/spack_repo/benchpark/packages/kripke/package.py index cc7a8714a..905b55ced 100644 --- a/repos/spack_repo/benchpark/packages/kripke/package.py +++ b/repos/spack_repo/benchpark/packages/kripke/package.py @@ -15,15 +15,15 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): """ homepage = "https://computing.llnl.gov/projects/co-design/kripke" - git = "https://github.com/LLNL/Kripke.git" + git = "https://github.com/rfhaque/Kripke.git" tags = ["proxy-app"] - maintainers("vsrana01") + maintainers("chen59") license("BSD-3-Clause") - version("develop", branch="develop", submodules=False) + version("develop", branch="kripke_chai_umpire", submodules=False) version("2025.12.0", submodules=False, commit="01f6f85c02ceffcd2bc06e42cee997867dd142c5") version("2025.07.0", submodules=False, commit="8cf38433a6a11e0dcd17864e649b2d045159ee9c") version( @@ -55,58 +55,99 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): ) variant("mpi", default=True, description="Build with MPI.") + variant("chai", default=True, description="Build with CHAI/Umpire.") + variant("direct-device-plane", default=False, description="Use direct device allocator in Umpire for plane fields") variant("openmp", default=False, description="Build with OpenMP enabled.") variant("caliper", default=False, description="Build with Caliper support enabled.") - variant("single_memory", default=False, description="Enable single memory space model in rocm") - - conflicts("+single_memory", when="~rocm") - depends_on("chai+single_memory", when="+single_memory") + variant("gpu-aware-mpi", default=False, description="Enable GPU-aware MPI") depends_on("c", type="build") depends_on("cxx", type="build") depends_on("fortran", type="build") - depends_on("chai@2025.12.0+raja", when="@develop") - depends_on("chai@2025.12.0+raja", when="@2025.12.0") - depends_on("fmt@9.1", when=f"^chai@2024.07.0") - depends_on("chai@2024.07.0+raja", when="@:2025.07.0") - depends_on("chai@2024.07.0+raja", when="@1.2.7.0:2025.07.0") - depends_on("fmt@9.1", when=f"^chai@2024.07.0") + depends_on("camp@2026.07.1", when="@develop") + depends_on("camp@2025.12.0", when="@2025.12.0") + depends_on("camp@2024.07.0", when="@1.2.7.0:2025.07.0") + + depends_on("raja@2026.07.0 ~examples~exercises cxxstd=20", when="@develop") + depends_on("raja@2025.12.0 ~examples~exercises cxxstd=17", when="@2025.12.0") + depends_on("raja@2024.07.0 ~examples~exercises cxxstd=14", when="@1.2.7.0:2025.07.0") depends_on("mpi", when="+mpi") - depends_on("chai+mpi", when="+mpi") + + with when("+chai"): + depends_on("chai+mpi", when="+mpi") + depends_on("chai@2026.07.0 +raja cxxstd=20", when="@develop") + depends_on("chai@2025.12.0 +raja cxxstd=17", when="@2025.12.0") + depends_on("chai@2024.07.0 +raja cxxstd=14", when="@1.2.7.0:2025.07.0") + + depends_on("fmt@9.1", when=f"^chai@2024.07.0") + + depends_on("chai+openmp", when="+openmp") + depends_on("chai~openmp", when="~openmp") + + depends_on("chai+cuda", when="+cuda") + depends_on("chai~cuda", when="~cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("chai cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + + depends_on("chai+rocm", when="+rocm") + depends_on("chai~rocm", when="~rocm") + for arch in ROCmPackage.amdgpu_targets: + depends_on("chai amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch)) + + depends_on("umpire@2026.07.1", when="@develop") + depends_on("umpire@2025.12.0", when="@2025.12.0") + depends_on("umpire@2024.07.0", when="@1.2.7.0:2025.07.0") + depends_on("umpire+openmp", when="+openmp") + depends_on("umpire~openmp", when="~openmp") + + depends_on("umpire+cuda", when="+cuda") + depends_on("umpire~cuda", when="~cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("umpire cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + + depends_on("umpire+rocm", when="+rocm") + depends_on("umpire~rocm", when="~rocm") + for arch in ROCmPackage.amdgpu_targets: + depends_on("umpire amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch)) + + conflicts("^blt@0.7:", when="^chai@:2024.07.0") + depends_on("caliper", when="+caliper") depends_on("adiak@0.4:", when="+caliper") conflicts("^blt@:0.3.6", when="+rocm") - conflicts("^blt@0.7:", when="^chai@:2024.07.0") depends_on("blt@0.6.2:", type="build", when=f"@1.2.7:") - depends_on("chai+openmp", when="+openmp") - depends_on("chai~openmp", when="~openmp") - - depends_on("chai+cuda", when="+cuda") - depends_on("chai~cuda", when="~cuda") + with when("+cuda~chai"): + depends_on("umpire@2026.07.1", when="@develop") + depends_on("umpire@2025.12.0", when="@2025.12.0") + depends_on("umpire@2024.07.0", when="@1.2.7.0:2025.07.0") + depends_on("umpire+cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("umpire cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + + with when("+rocm~chai"): + depends_on("umpire@2026.07.1", when="@develop") + depends_on("umpire@2025.12.0", when="@2025.12.0") + depends_on("umpire@2024.07.0", when="@1.2.7.0:2025.07.0") + depends_on("umpire+rocm") + for arch in ROCmPackage.amdgpu_targets: + depends_on("umpire amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch)) + + depends_on("raja+openmp", when="+openmp") + depends_on("raja~openmp", when="~openmp") + + depends_on("raja+cuda", when="+cuda") + depends_on("raja~cuda", when="~cuda") for sm_ in CudaPackage.cuda_arch_values: - depends_on("chai cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + depends_on("raja cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) - depends_on("chai+rocm", when="+rocm") - depends_on("chai~rocm", when="~rocm") + depends_on("raja+rocm", when="+rocm") + depends_on("raja~rocm", when="~rocm") for arch in ROCmPackage.amdgpu_targets: - depends_on("chai amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch)) - - depends_on("umpire+openmp", when="+openmp") - depends_on("umpire~openmp", when="~openmp") - - depends_on("umpire+cuda", when="+cuda") - depends_on("umpire~cuda", when="~cuda") - for sm_ in CudaPackage.cuda_arch_values: - depends_on("umpire cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) - - depends_on("umpire+rocm", when="+rocm") - depends_on("umpire~rocm", when="~rocm") - for arch in ROCmPackage.amdgpu_targets: - depends_on("umpire amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch)) + depends_on("raja amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch)) def setup_build_environment(self, env): spec = self.spec @@ -123,52 +164,56 @@ def setup_run_environment(self, env): def cmake_args(self): spec = self.spec args = [] - - if "+rocm" in spec or "+cuda" in spec: - enable_chai = "ON" - enable_chai_single_memory = "ON" if "+single_memory" in spec else "OFF" - else: - enable_chai = "OFF" - enable_chai_single_memory = "OFF" + if spec.satisfies("@develop"): + blt_cxx_std = "c++20" + elif spec.satisfies("@2025.12.0"): + blt_cxx_std = "c++17" + elif spec.satisfies("@:2025.7.0"): + blt_cxx_std = "c++14" args.extend( [ - "-DCAMP_DIR=%s" % self.spec["camp"].prefix, + "-Dcamp_DIR=%s" % self.spec["camp"].prefix, "-DBLT_SOURCE_DIR=%s" % self.spec["blt"].prefix, - "-Dumpire_DIR=%s" % self.spec["umpire"].prefix, "-DRAJA_DIR=%s" % self.spec["raja"].prefix, - "-Dchai_DIR=%s" % self.spec["chai"].prefix, - "-DENABLE_CHAI=%s" % enable_chai, - "-DENABLE_CHAI_SINGLE_MEMORY=%s" % enable_chai_single_memory, + "-DBLT_CXX_STD=%s" % blt_cxx_std, "-DMPI_CXX_LINK_FLAGS='%s'" % self.spec['mpi'].libs.ld_flags, ] ) - if "+openmp" in spec: - args.append("-DENABLE_OPENMP=ON") + args.append(self.define_from_variant("ENABLE_CHAI", "chai")) + if "+chai" in spec: + args.extend( + [ + "-Dchai_DIR=%s" % self.spec["chai"].prefix, + "-Dumpire_DIR=%s" % self.spec["umpire"].prefix + "/lib64/cmake/umpire", + ] + ) - if "+caliper" in spec: - args.append("-DENABLE_CALIPER=ON") + args.append(self.define_from_variant("ENABLE_GPU_AWARE_MPI", "gpu-aware-mpi")) + if "+gpu-aware-mpi" in spec: + args.append(self.define_from_variant("ENABLE_DIRECT_UMPIRE_PLANE_STORAGE", "direct-device-plane")) + args.append(self.define_from_variant("ENABLE_OPENMP", "openmp")) + args.append(self.define_from_variant("ENABLE_CALIPER", "caliper")) + args.append(self.define_from_variant("ENABLE_MPI", "mpi")) if "+mpi" in spec: - args.append("-DENABLE_MPI=ON") args.append(self.define("CMAKE_CXX_COMPILER", self.spec["mpi"].mpicxx)) + args.append(self.define_from_variant("ENABLE_HIP", "rocm")) if "+rocm" in spec: # Set up the hip macros needed by the build - args.append("-DENABLE_HIP=ON") + args.append("-Dumpire_DIR=%s" % self.spec["umpire"].prefix + "/lib64/cmake/umpire") args.append("-DHIP_ROOT_DIR={0}".format(spec["hip"].prefix)) rocm_archs = spec.variants["amdgpu_target"].value if "none" not in rocm_archs: arch_str = ",".join(rocm_archs) args.append("-DHIP_HIPCC_FLAGS=--amdgpu-target={0}".format(arch_str)) args.append("-DCMAKE_HIP_ARCHITECTURES={0}".format(arch_str)) - else: - # Ensure build with hip is disabled - args.append("-DENABLE_HIP=OFF") + args.append(self.define_from_variant("ENABLE_CUDA", "cuda")) if "+cuda" in spec: - args.append("-DENABLE_CUDA=ON") + args.append("-Dumpire_DIR=%s" % self.spec["umpire"].prefix + "/lib64/cmake/umpire") args.append(self.define("CMAKE_CUDA_HOST_COMPILER", self.spec["mpi"].mpicxx)) if not spec.satisfies("cuda_arch=none"): cuda_arch = spec.variants["cuda_arch"].value @@ -178,8 +223,6 @@ def cmake_args(self): "-DCMAKE_CUDA_FLAGS=--extended-lambda -I=%s" % (self.spec["mpi"].prefix.include) ) - else: - args.append("-DENABLE_CUDA=OFF") return args diff --git a/repos/spack_repo/benchpark/packages/raja/package.py b/repos/spack_repo/benchpark/packages/raja/package.py deleted file mode 100644 index b504e8f21..000000000 --- a/repos/spack_repo/benchpark/packages/raja/package.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -from spack.package import * -from spack_repo.builtin.packages.raja.package import Raja as BuiltinRaja - - -class Raja(BuiltinRaja): - - version( - "2025.12.2", - tag="v2025.12.2", - commit="eca7c5015a5cf8bf7cc8ad1829fd36d3276ab274", - submodules=False, - ) - version( - "2025.12.1", - tag="v2025.12.1", - commit="3b8b59a1e9be2e1066c0d77372b3bf5956e6d6e2", - submodules=False, - ) - version( - "2025.12.0", - tag="v2025.12.0", - commit="e827035c630e71a9358e2f21c2f3cf6fd5fb6605", - submodules=False, - ) diff --git a/repos/spack_repo/benchpark/packages/umpire/package.py b/repos/spack_repo/benchpark/packages/umpire/package.py deleted file mode 100644 index 1b0562d7e..000000000 --- a/repos/spack_repo/benchpark/packages/umpire/package.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -from spack.package import * -from spack_repo.builtin.packages.umpire.package import Umpire as BuiltinUmpire - - -class Umpire(BuiltinUmpire): - - version( - "2025.12.0", - tag="v2025.12.0", - commit="0372fbd6e1f17d7e6dd72693f8b857f3ec7559e9", - submodules=False, - ) - - depends_on("fmt@9.1: cxxstd=17", when="@2024.02.0: %oneapi") diff --git a/systems/common/spack_includes.yaml b/systems/common/spack_includes.yaml index 7a034fb2c..d3588e678 100644 --- a/systems/common/spack_includes.yaml +++ b/systems/common/spack_includes.yaml @@ -2,7 +2,4 @@ # Benchpark Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: Apache-2.0 -packages: - blt: - require: - - one_of: ['%gcc', '%clang'] +packages: {}