From 58934002618e2971477902d2416459a181066135 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Sep 2026 21:09:21 -0600 Subject: [PATCH 1/4] Update Blueback site config: PrgEnv-*/8.7.0 --- configs/sites/tier1/blueback/packages_gcc-14.3.0.yaml | 2 +- configs/sites/tier1/blueback/packages_oneapi-2026.0.0.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/sites/tier1/blueback/packages_gcc-14.3.0.yaml b/configs/sites/tier1/blueback/packages_gcc-14.3.0.yaml index 0d4df8a20..65ac67f20 100644 --- a/configs/sites/tier1/blueback/packages_gcc-14.3.0.yaml +++ b/configs/sites/tier1/blueback/packages_gcc-14.3.0.yaml @@ -13,7 +13,7 @@ packages: - spec: gcc@14.3.0 languages:='c,c++,fortran' prefix: /usr modules: - - PrgEnv-gnu/8.6.0 + - PrgEnv-gnu/8.7.0 - gcc-native/14.2 extra_attributes: compilers: diff --git a/configs/sites/tier1/blueback/packages_oneapi-2026.0.0.yaml b/configs/sites/tier1/blueback/packages_oneapi-2026.0.0.yaml index 490039112..5683c7705 100644 --- a/configs/sites/tier1/blueback/packages_oneapi-2026.0.0.yaml +++ b/configs/sites/tier1/blueback/packages_oneapi-2026.0.0.yaml @@ -15,7 +15,7 @@ packages: - spec: intel-oneapi-compilers@2026.0.0 +fix_rt_linkage prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2026.0.0 modules: - - PrgEnv-intel/8.6.0 + - PrgEnv-intel/8.7.0 - intel-oneapi/2026.0.0 extra_attributes: compilers: @@ -31,7 +31,7 @@ packages: - spec: gcc@13.3.0 languages:='c,c++' prefix: /usr modules: - - PrgEnv-gnu/8.6.0 + - PrgEnv-gnu/8.7.0 - gcc-native/13.2 extra_attributes: compilers: From 4d784d0f14a98cdb73265b5ad4c82ff50f5ef9e7 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 23 Sep 2026 10:16:08 -0600 Subject: [PATCH 2/4] configs/common/packages.yaml: either py-numpy@2.4.6 + py-pandas ~performance (preferred/default), or py-numpy@2.2.2 + py-pandas +performance (require in site config) --- configs/common/packages.yaml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 6a1d84d3b..8789fb23f 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -467,14 +467,21 @@ packages: py-netcdf4: require: - ~mpi - # To avoid duplicate packages + # To avoid duplicate packages - use either 2.4.6 or 2.3.3 (needed for py-pandas +performace) py-numpy: require: - - '@=2.4.6' - # Restrict py-pandas to older versions compatible with py-numpy and py-xarray + - one_of: + - '@2.4.6' + - '@2.3.3' + prefer: + - '@2.4.6' + # Restrict py-pandas to versions 2 and older. Users wanting the +performance variant + # must 'require' it in their site config. py-pandas: require: - '@:2' + prefer: + - '~performance' # py-versioneer is unavoidably duplicated by pins in py-pyogrio (@0.28) and # py-partd (@0.29). py-pandas is unconstrained and can further cascade splits # into py-xarray/met/metplus/jedi-base-env. This constraints py-versioneer to @0.28 From 2db2f4f803c2b4a12ecbb884631fb0e604b7931a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 23 Sep 2026 10:22:28 -0600 Subject: [PATCH 3/4] Apply suggestion from @climbfuji --- configs/common/packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 8789fb23f..5a603b534 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -467,7 +467,7 @@ packages: py-netcdf4: require: - ~mpi - # To avoid duplicate packages - use either 2.4.6 or 2.3.3 (needed for py-pandas +performace) + # To avoid duplicate packages - use either 2.4.6 or 2.3.3 (needed for py-pandas +performance) py-numpy: require: - one_of: From d55f8a85f2991a21c2a654c1e136b618c81fc7a5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 23 Sep 2026 12:08:42 -0600 Subject: [PATCH 4/4] Make spack stack check-preferred-compiler work with any_of, one_of, ... --- .../spack-stack/stack/compiler_utils.py | 74 ++++++++++--------- .../tests/test_check_preferred_compiler.py | 62 +++++++++++++++- 2 files changed, 100 insertions(+), 36 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/compiler_utils.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/compiler_utils.py index e3343a816..7a8acd888 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/compiler_utils.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/compiler_utils.py @@ -2,6 +2,7 @@ import logging import re +from collections.abc import Mapping import spack import spack.environment as ev @@ -49,6 +50,18 @@ def get_compiler_choice(string): return None +def iter_constraint_strings(value): + """Recursively flatten nested Spack config values like one_of/any_of.""" + if isinstance(value, (str, bytes)): + yield value + elif isinstance(value, Mapping): + for nested in value.values(): + yield from iter_constraint_strings(nested) + elif isinstance(value, (list, tuple)): + for nested in value: + yield from iter_constraint_strings(nested) + + def check_preferred_compiler(): """For an active environment, check that the preferred compiler is being used for all packages except those that explicitly @@ -112,46 +125,41 @@ def check_preferred_compiler(): spec_required_compiler_version = None spec_preferred_compiler_name = None spec_preferred_compiler_version = None + spec_required_compilers = [] + spec_preferred_compilers = [] for key, value in package_config[spec.name].items(): - # To simplify parsing, turn scalar values into CommentedSeq of length 1 - if isinstance(value, (str, bytes)): - values = CommentedSeq([value]) - else: - values = value - # Loop through all values to check for required or preferred compilers - for entry in values: + if key.lower() not in ("require", "prefer"): + continue + for entry in iter_constraint_strings(value): + if not isinstance(entry, (str, bytes)): + continue + entry = entry.lower() + choice = get_compiler_choice(entry) + if not choice: + continue if key.lower() == "require": - choice = get_compiler_choice(entry.lower()) - # Not a compiler preference, carry on - if not choice: - continue - # Check that the explicitly required compiler is a valid (existing) - # compiler for this environment. This requirement may be relaxed in - # the future if we start building compilers in spack environments. if any(choice in c for c in compilers): (spec_required_compiler_name, spec_required_compiler_version) = get_compiler_name_and_version(choice) + spec_required_compilers.append((spec_required_compiler_name, spec_required_compiler_version)) elif key.lower() == "prefer": - choice = get_compiler_choice(entry.lower()) - # Not a compiler preference, carry on - if not choice: - continue - # Check that the explicitly preferred compiler is a valid (existing) - # compiler for this environment. This requirement may be relaxed in - # the future if we start building compilers in spack environments. if any(choice in c for c in compilers): (spec_preferred_compiler_name, spec_preferred_compiler_version) = get_compiler_name_and_version(choice) - # If we have a hard requirement for a compiler, we can stop scanning the spec package config - if spec_required_compiler_name: - break - if spec_required_compiler_name == compiler_name and \ - ( (not spec_required_compiler_version or not compiler_version) or \ - (spec_required_compiler_version==compiler_version) ): - logging.info(f" ... {spec.name}@{spec.version}/{spec.dag_hash(length=7)} uses explicitly required compiler") - elif spec_preferred_compiler_name == compiler_name and \ - ( (not spec_preferred_compiler_version or not compiler_version) or \ - (spec_preferred_compiler_version==compiler_version) ): - logging.info(f" ... {spec.name}@{spec.version}/{spec.dag_hash(length=7)} uses explicitly preferred compiler") - else: + spec_preferred_compilers.append((spec_preferred_compiler_name, spec_preferred_compiler_version)) + ok = False + for (spec_required_compiler_name, spec_required_compiler_version) in spec_required_compilers: + if spec_required_compiler_name == compiler_name and \ + ( (not spec_required_compiler_version or not compiler_version) or \ + (spec_required_compiler_version==compiler_version) ): + logging.info(f" ... {spec.name}@{spec.version}/{spec.dag_hash(length=7)} uses explicitly required compiler") + ok = True + if not ok: + for (spec_preferred_compiler_name, spec_preferred_compiler_version) in spec_preferred_compilers: + if spec_preferred_compiler_name == compiler_name and \ + ( (not spec_preferred_compiler_version or not compiler_version) or \ + (spec_preferred_compiler_version==compiler_version) ): + logging.info(f" ... {spec.name}@{spec.version}/{spec.dag_hash(length=7)} uses explicitly preferred compiler") + ok = True + if not ok: errors += 1 logging.error(f" ... {RED}error: {spec.name}@{spec.version}/{spec.dag_hash(length=7)} does not use intended compiler{RESET}") if errors==1: diff --git a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_check_preferred_compiler.py b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_check_preferred_compiler.py index 6980d431c..f6ad436de 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_check_preferred_compiler.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_check_preferred_compiler.py @@ -78,22 +78,78 @@ def test_check_preferred_compiler(): libszip: require: - '%c=gcc' + xz: + require: + - one_of: + - '%c=gcc' + - '%c=llvm' + gmake: + require: + - any_of: + - '%c=gcc' + - '%c=llvm' """ site_packages_yaml = os.path.join(env_dir, "site", "packages.yaml") if os.path.exists(site_packages_yaml): - raise Exception("Not implemented: appending to existing {site_packages_yaml}") + os.remove(site_packages_yaml) with open(site_packages_yaml, 'w') as f: f.write(packages_definition) cmd = spack.main.SpackCommand("add") - cmd("gcc", "openmpi", "zlib", "libszip") + cmd("gcc", "openmpi", "zlib", "libszip", "xz", "gmake") cmd = spack.main.SpackCommand("concretize") cmd("--force", "--fresh") spack_stack_cmd("check-preferred-compiler") - filter_file("%c=gcc", "%c=llvm", site_packages_yaml, string=True) + packages_definition = """ +packages: + gcc: + externals: + - spec: gcc@11.5.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + llvm: + externals: + - spec: llvm@21.1.0 + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ + fortran: /usr/bin/flang-new + mpi: + buildable: false + openmpi: + externals: + - spec: openmpi@5.0.8 ~internal-hwloc +two_level_namespace + prefix: /usr + zlib: + prefer: + - '%c=llvm' + libszip: + require: + - '%c=llvm' + xz: + require: + - one_of: + - '%c=gcc' + - '%c=llvm' + gmake: + require: + - any_of: + - '%c=gcc' + - '%c=llvm' +""" + if os.path.exists(site_packages_yaml): + os.remove(site_packages_yaml) + with open(site_packages_yaml, 'w') as f: + f.write(packages_definition) with pytest.raises(Exception) as error: spack_stack_cmd("check-preferred-compiler")