diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 6a1d84d3b..5a603b534 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -467,14 +467,21 @@ packages: py-netcdf4: require: - ~mpi - # To avoid duplicate packages + # To avoid duplicate packages - use either 2.4.6 or 2.3.3 (needed for py-pandas +performance) py-numpy: require: - - '@=2.4.6' - # Restrict py-pandas to older versions compatible with py-numpy and py-xarray + - one_of: + - '@2.4.6' + - '@2.3.3' + prefer: + - '@2.4.6' + # Restrict py-pandas to versions 2 and older. Users wanting the +performance variant + # must 'require' it in their site config. py-pandas: require: - '@:2' + prefer: + - '~performance' # py-versioneer is unavoidably duplicated by pins in py-pyogrio (@0.28) and # py-partd (@0.29). py-pandas is unconstrained and can further cascade splits # into py-xarray/met/metplus/jedi-base-env. This constraints py-versioneer to @0.28 diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/compiler_utils.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/compiler_utils.py index e3343a816..7a8acd888 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/compiler_utils.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/compiler_utils.py @@ -2,6 +2,7 @@ import logging import re +from collections.abc import Mapping import spack import spack.environment as ev @@ -49,6 +50,18 @@ def get_compiler_choice(string): return None +def iter_constraint_strings(value): + """Recursively flatten nested Spack config values like one_of/any_of.""" + if isinstance(value, (str, bytes)): + yield value + elif isinstance(value, Mapping): + for nested in value.values(): + yield from iter_constraint_strings(nested) + elif isinstance(value, (list, tuple)): + for nested in value: + yield from iter_constraint_strings(nested) + + def check_preferred_compiler(): """For an active environment, check that the preferred compiler is being used for all packages except those that explicitly @@ -112,46 +125,41 @@ def check_preferred_compiler(): spec_required_compiler_version = None spec_preferred_compiler_name = None spec_preferred_compiler_version = None + spec_required_compilers = [] + spec_preferred_compilers = [] for key, value in package_config[spec.name].items(): - # To simplify parsing, turn scalar values into CommentedSeq of length 1 - if isinstance(value, (str, bytes)): - values = CommentedSeq([value]) - else: - values = value - # Loop through all values to check for required or preferred compilers - for entry in values: + if key.lower() not in ("require", "prefer"): + continue + for entry in iter_constraint_strings(value): + if not isinstance(entry, (str, bytes)): + continue + entry = entry.lower() + choice = get_compiler_choice(entry) + if not choice: + continue if key.lower() == "require": - choice = get_compiler_choice(entry.lower()) - # Not a compiler preference, carry on - if not choice: - continue - # Check that the explicitly required compiler is a valid (existing) - # compiler for this environment. This requirement may be relaxed in - # the future if we start building compilers in spack environments. if any(choice in c for c in compilers): (spec_required_compiler_name, spec_required_compiler_version) = get_compiler_name_and_version(choice) + spec_required_compilers.append((spec_required_compiler_name, spec_required_compiler_version)) elif key.lower() == "prefer": - choice = get_compiler_choice(entry.lower()) - # Not a compiler preference, carry on - if not choice: - continue - # Check that the explicitly preferred compiler is a valid (existing) - # compiler for this environment. This requirement may be relaxed in - # the future if we start building compilers in spack environments. if any(choice in c for c in compilers): (spec_preferred_compiler_name, spec_preferred_compiler_version) = get_compiler_name_and_version(choice) - # If we have a hard requirement for a compiler, we can stop scanning the spec package config - if spec_required_compiler_name: - break - if spec_required_compiler_name == compiler_name and \ - ( (not spec_required_compiler_version or not compiler_version) or \ - (spec_required_compiler_version==compiler_version) ): - logging.info(f" ... {spec.name}@{spec.version}/{spec.dag_hash(length=7)} uses explicitly required compiler") - elif spec_preferred_compiler_name == compiler_name and \ - ( (not spec_preferred_compiler_version or not compiler_version) or \ - (spec_preferred_compiler_version==compiler_version) ): - logging.info(f" ... {spec.name}@{spec.version}/{spec.dag_hash(length=7)} uses explicitly preferred compiler") - else: + spec_preferred_compilers.append((spec_preferred_compiler_name, spec_preferred_compiler_version)) + ok = False + for (spec_required_compiler_name, spec_required_compiler_version) in spec_required_compilers: + if spec_required_compiler_name == compiler_name and \ + ( (not spec_required_compiler_version or not compiler_version) or \ + (spec_required_compiler_version==compiler_version) ): + logging.info(f" ... {spec.name}@{spec.version}/{spec.dag_hash(length=7)} uses explicitly required compiler") + ok = True + if not ok: + for (spec_preferred_compiler_name, spec_preferred_compiler_version) in spec_preferred_compilers: + if spec_preferred_compiler_name == compiler_name and \ + ( (not spec_preferred_compiler_version or not compiler_version) or \ + (spec_preferred_compiler_version==compiler_version) ): + logging.info(f" ... {spec.name}@{spec.version}/{spec.dag_hash(length=7)} uses explicitly preferred compiler") + ok = True + if not ok: errors += 1 logging.error(f" ... {RED}error: {spec.name}@{spec.version}/{spec.dag_hash(length=7)} does not use intended compiler{RESET}") if errors==1: diff --git a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_check_preferred_compiler.py b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_check_preferred_compiler.py index 6980d431c..f6ad436de 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_check_preferred_compiler.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_check_preferred_compiler.py @@ -78,22 +78,78 @@ def test_check_preferred_compiler(): libszip: require: - '%c=gcc' + xz: + require: + - one_of: + - '%c=gcc' + - '%c=llvm' + gmake: + require: + - any_of: + - '%c=gcc' + - '%c=llvm' """ site_packages_yaml = os.path.join(env_dir, "site", "packages.yaml") if os.path.exists(site_packages_yaml): - raise Exception("Not implemented: appending to existing {site_packages_yaml}") + os.remove(site_packages_yaml) with open(site_packages_yaml, 'w') as f: f.write(packages_definition) cmd = spack.main.SpackCommand("add") - cmd("gcc", "openmpi", "zlib", "libszip") + cmd("gcc", "openmpi", "zlib", "libszip", "xz", "gmake") cmd = spack.main.SpackCommand("concretize") cmd("--force", "--fresh") spack_stack_cmd("check-preferred-compiler") - filter_file("%c=gcc", "%c=llvm", site_packages_yaml, string=True) + packages_definition = """ +packages: + gcc: + externals: + - spec: gcc@11.5.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + llvm: + externals: + - spec: llvm@21.1.0 + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ + fortran: /usr/bin/flang-new + mpi: + buildable: false + openmpi: + externals: + - spec: openmpi@5.0.8 ~internal-hwloc +two_level_namespace + prefix: /usr + zlib: + prefer: + - '%c=llvm' + libszip: + require: + - '%c=llvm' + xz: + require: + - one_of: + - '%c=gcc' + - '%c=llvm' + gmake: + require: + - any_of: + - '%c=gcc' + - '%c=llvm' +""" + if os.path.exists(site_packages_yaml): + os.remove(site_packages_yaml) + with open(site_packages_yaml, 'w') as f: + f.write(packages_definition) with pytest.raises(Exception) as error: spack_stack_cmd("check-preferred-compiler")