Choose between py-numpy@2.4.6 + py-pandas~performance or py-numpy@2.3.3 + py-pandas+performance

configs/common/packages.yaml

@@ -467,14 +467,21 @@ packages:
     py-netcdf4:
       require:
       - ~mpi
-    # To avoid duplicate packages
+    # To avoid duplicate packages - use either 2.4.6 or 2.3.3 (needed for py-pandas +performance)
     py-numpy:
       require:
-      - '@=2.4.6'
-    # Restrict py-pandas to older versions compatible with py-numpy and py-xarray
+      - one_of:
+        - '@2.4.6'
+        - '@2.3.3'
+      prefer:
+      - '@2.4.6'
+    # Restrict py-pandas to versions 2 and older. Users wanting the +performance variant
+    # must 'require' it in their site config.
     py-pandas:
       require:
       - '@:2'
+      prefer:
+      - '~performance'
       # py-versioneer is unavoidably duplicated by pins in py-pyogrio (@0.28) and
       # py-partd (@0.29). py-pandas is unconstrained and can further cascade splits
       # into py-xarray/met/metplus/jedi-base-env. This constraints py-versioneer to @0.28

spack-ext/lib/jcsda-emc/spack-stack/stack/compiler_utils.py

@@ -2,6 +2,7 @@
 
 import logging
 import re
+from collections.abc import Mapping
 
 import spack
 import spack.environment as ev
@@ -49,6 +50,18 @@ def get_compiler_choice(string):
     return None
 
 
+def iter_constraint_strings(value):
+    """Recursively flatten nested Spack config values like one_of/any_of."""
+    if isinstance(value, (str, bytes)):
+        yield value
+    elif isinstance(value, Mapping):
+        for nested in value.values():
+            yield from iter_constraint_strings(nested)
+    elif isinstance(value, (list, tuple)):
+        for nested in value:
+            yield from iter_constraint_strings(nested)
+
+
 def check_preferred_compiler():
     """For an active environment, check that the preferred compiler
     is being used for all packages except those that explicitly
@@ -112,46 +125,41 @@ def check_preferred_compiler():
             spec_required_compiler_version = None
             spec_preferred_compiler_name = None
             spec_preferred_compiler_version = None
+            spec_required_compilers = []
+            spec_preferred_compilers = []
             for key, value in package_config[spec.name].items():
-                # To simplify parsing, turn scalar values into CommentedSeq of length 1
-                if isinstance(value, (str, bytes)):
-                    values = CommentedSeq([value])
-                else:
-                    values = value
-                # Loop through all values to check for required or preferred compilers
-                for entry in values:
+                if key.lower() not in ("require", "prefer"):
+                    continue
+                for entry in iter_constraint_strings(value):
+                    if not isinstance(entry, (str, bytes)):
+                        continue
+                    entry = entry.lower()
+                    choice = get_compiler_choice(entry)
+                    if not choice:
+                        continue
                     if key.lower() == "require":
-                        choice = get_compiler_choice(entry.lower())
-                        # Not a compiler preference, carry on
-                        if not choice:
-                            continue
-                        # Check that the explicitly required compiler is a valid (existing)
-                        # compiler for this environment. This requirement may be relaxed in
-                        # the future if we start building compilers in spack environments.
                         if any(choice in c for c in compilers):
                             (spec_required_compiler_name, spec_required_compiler_version) = get_compiler_name_and_version(choice)
+                            spec_required_compilers.append((spec_required_compiler_name, spec_required_compiler_version))
                     elif key.lower() == "prefer":
-                        choice = get_compiler_choice(entry.lower())
-                        # Not a compiler preference, carry on
-                        if not choice:
-                            continue
-                        # Check that the explicitly preferred compiler is a valid (existing)
-                        # compiler for this environment. This requirement may be relaxed in
-                        # the future if we start building compilers in spack environments.
                         if any(choice in c for c in compilers):
                             (spec_preferred_compiler_name, spec_preferred_compiler_version) = get_compiler_name_and_version(choice)
-                # If we have a hard requirement for a compiler, we can stop scanning the spec package config
-                if spec_required_compiler_name:
-                    break
-            if spec_required_compiler_name == compiler_name and \
-                ( (not spec_required_compiler_version or not compiler_version) or \
-                  (spec_required_compiler_version==compiler_version) ):
-                logging.info(f"  ... {spec.name}@{spec.version}/{spec.dag_hash(length=7)} uses explicitly required compiler")
-            elif spec_preferred_compiler_name == compiler_name and \
-                ( (not spec_preferred_compiler_version or not compiler_version) or \
-                  (spec_preferred_compiler_version==compiler_version) ):
-                logging.info(f"  ... {spec.name}@{spec.version}/{spec.dag_hash(length=7)} uses explicitly preferred compiler")
-            else:
+                            spec_preferred_compilers.append((spec_preferred_compiler_name, spec_preferred_compiler_version))
+            ok = False
+            for (spec_required_compiler_name, spec_required_compiler_version) in spec_required_compilers:
+                if spec_required_compiler_name == compiler_name and \
+                    ( (not spec_required_compiler_version or not compiler_version) or \
+                      (spec_required_compiler_version==compiler_version) ):
+                    logging.info(f"  ... {spec.name}@{spec.version}/{spec.dag_hash(length=7)} uses explicitly required compiler")
+                    ok = True
+            if not ok:
+                for (spec_preferred_compiler_name, spec_preferred_compiler_version) in spec_preferred_compilers:
+                    if spec_preferred_compiler_name == compiler_name and \
+                        ( (not spec_preferred_compiler_version or not compiler_version) or \
+                          (spec_preferred_compiler_version==compiler_version) ):
+                        logging.info(f"  ... {spec.name}@{spec.version}/{spec.dag_hash(length=7)} uses explicitly preferred compiler")
+                        ok = True
+            if not ok:
                 errors += 1
                 logging.error(f"  ... {RED}error: {spec.name}@{spec.version}/{spec.dag_hash(length=7)} does not use intended compiler{RESET}")
     if errors==1:

spack-ext/lib/jcsda-emc/spack-stack/tests/test_check_preferred_compiler.py

@@ -78,22 +78,78 @@ def test_check_preferred_compiler():
   libszip:
     require:
     - '%c=gcc'
+  xz:
+    require:
+    - one_of:
+      - '%c=gcc'
+      - '%c=llvm'
+  gmake:
+    require:
+    - any_of:
+      - '%c=gcc'
+      - '%c=llvm'
 """
     site_packages_yaml = os.path.join(env_dir, "site", "packages.yaml")
     if os.path.exists(site_packages_yaml):
-        raise Exception("Not implemented: appending to existing {site_packages_yaml}")
+        os.remove(site_packages_yaml)
     with open(site_packages_yaml, 'w') as f:
         f.write(packages_definition)
 
     cmd = spack.main.SpackCommand("add")
-    cmd("gcc", "openmpi", "zlib", "libszip")
+    cmd("gcc", "openmpi", "zlib", "libszip", "xz", "gmake")
 
     cmd = spack.main.SpackCommand("concretize")
     cmd("--force", "--fresh")
 
     spack_stack_cmd("check-preferred-compiler")
 
-    filter_file("%c=gcc", "%c=llvm", site_packages_yaml, string=True)
+    packages_definition = """
+packages:
+  gcc:
+    externals:
+    - spec: gcc@11.5.0 languages:='c,c++,fortran'
+      prefix: /usr
+      extra_attributes:
+        compilers:
+          c: /usr/bin/gcc
+          cxx: /usr/bin/g++
+          fortran: /usr/bin/gfortran
+  llvm:
+    externals:
+    - spec: llvm@21.1.0
+      prefix: /usr
+      extra_attributes:
+        compilers:
+          c: /usr/bin/clang
+          cxx: /usr/bin/clang++
+          fortran: /usr/bin/flang-new
+  mpi:
+    buildable: false
+  openmpi:
+    externals:
+    - spec: openmpi@5.0.8 ~internal-hwloc +two_level_namespace
+      prefix: /usr
+  zlib:
+    prefer:
+    - '%c=llvm'
+  libszip:
+    require:
+    - '%c=llvm'
+  xz:
+    require:
+    - one_of:
+      - '%c=gcc'
+      - '%c=llvm'
+  gmake:
+    require:
+    - any_of:
+      - '%c=gcc'
+      - '%c=llvm'
+"""
+    if os.path.exists(site_packages_yaml):
+        os.remove(site_packages_yaml)
+    with open(site_packages_yaml, 'w') as f:
+        f.write(packages_definition)
 
     with pytest.raises(Exception) as error:
         spack_stack_cmd("check-preferred-compiler")