diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index e55a94bd38..7c45e79571 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -4,6 +4,8 @@ set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/install_rattler_build.sh" + source rapids-configure-sccache source rapids-datetime-string diff --git a/ci/build_python.sh b/ci/build_python.sh index feb5390bb4..c82dcd93ce 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -4,6 +4,8 @@ set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/install_rattler_build.sh" + source rapids-configure-sccache source rapids-datetime-string diff --git a/ci/install_rattler_build.sh b/ci/install_rattler_build.sh new file mode 100644 index 0000000000..cc9c2abc12 --- /dev/null +++ b/ci/install_rattler_build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Resolve an exact version so an incompatible solve cannot silently retain the +# older rattler-build installed in the CI image. +RATTLER_BUILD_VERSION=$( + conda search --json --override-channels -c conda-forge rattler-build | + python -c ' +import json +import sys +from packaging.version import Version + +versions = [Version(record["version"]) for record in json.load(sys.stdin)["rattler-build"]] +print(max(version for version in versions if not version.is_prerelease and not version.is_devrelease)) +' +) + +rapids-conda-retry install -y -c conda-forge "rattler-build=${RATTLER_BUILD_VERSION}" + +RATTLER_BUILD_INSTALLED_VERSION=$(rattler-build --version) +if [[ "${RATTLER_BUILD_INSTALLED_VERSION}" != "rattler-build ${RATTLER_BUILD_VERSION}" ]]; then + echo "Expected rattler-build ${RATTLER_BUILD_VERSION}, found ${RATTLER_BUILD_INSTALLED_VERSION}" >&2 + exit 1 +fi +echo "${RATTLER_BUILD_INSTALLED_VERSION}" diff --git a/conda/recipes/cuvs-bench-cpu/recipe.yaml b/conda/recipes/cuvs-bench-cpu/recipe.yaml index 76c4e7eb16..3c3911e8b4 100644 --- a/conda/recipes/cuvs-bench-cpu/recipe.yaml +++ b/conda/recipes/cuvs-bench-cpu/recipe.yaml @@ -48,8 +48,10 @@ build: NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }} RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }} + RAPIDS_CUDA_VERSION: ${{ env.get("RAPIDS_CUDA_VERSION") }} RAPIDS_PY_VERSION: ${{ py_abi_min }} SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} + SCCACHE_CONF: ${{ env.get("SCCACHE_CONF", default="") }} SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }} SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }} SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }} @@ -89,6 +91,9 @@ requirements: - libboost-devel=1.87 - mkl-devel=2023 run: + - if: aarch64 + then: libopenblas + - libgomp - click - glog ${{ glog_version }} - h5py ${{ h5py_version }} @@ -103,6 +108,7 @@ requirements: - if: linux64 then: - mkl =2023 + - llvm-openmp ignore_run_exports: by_name: - glog diff --git a/conda/recipes/cuvs/recipe.yaml b/conda/recipes/cuvs/recipe.yaml index fb19067a3c..8f90e7c174 100644 --- a/conda/recipes/cuvs/recipe.yaml +++ b/conda/recipes/cuvs/recipe.yaml @@ -40,8 +40,10 @@ build: NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }} RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }} + RAPIDS_CUDA_VERSION: ${{ env.get("RAPIDS_CUDA_VERSION") }} RAPIDS_PY_VERSION: ${{ env.get("RAPIDS_PY_VERSION") }} SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} + SCCACHE_CONF: ${{ env.get("SCCACHE_CONF", default="") }} SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }} SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }} SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }} diff --git a/conda/recipes/libcuvs/recipe.yaml b/conda/recipes/libcuvs/recipe.yaml index 9df66d7ff2..7b86445b8e 100644 --- a/conda/recipes/libcuvs/recipe.yaml +++ b/conda/recipes/libcuvs/recipe.yaml @@ -14,97 +14,101 @@ context: recipe: name: libcuvs-split -cache: - source: - path: ../../.. +source: + path: ../../.. - build: - script: - content: | +outputs: + - staging: + name: libcuvs-build + build: + script: + content: | - # Remove `-fdebug-prefix-map` line from CFLAGS and CXXFLAGS so the - # incrementing version number in the compile line doesn't break the - # cache - set -x - export CFLAGS=$(echo $CFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') - export CXXFLAGS=$(echo $CXXFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') - set +x + # Remove `-fdebug-prefix-map` line from CFLAGS and CXXFLAGS so the + # incrementing version number in the compile line doesn't break the + # cache + set -x + export CFLAGS=$(echo $CFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') + export CXXFLAGS=$(echo $CXXFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') + set +x - ./build.sh libcuvs bench-ann tests --allgpuarch --mnmg-tests --build-metrics=compile_lib --incl-cache-stats --no-nvtx -n + ./build.sh libcuvs bench-ann tests --allgpuarch --mnmg-tests --build-metrics=compile_lib --incl-cache-stats --no-nvtx -n - cmake --build cpp/build --target cuvs cuvs_c - ./examples/build.sh --allgpuarch + cmake --build cpp/build --target cuvs cuvs_c + ./examples/build.sh --allgpuarch - secrets: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - SCCACHE_DIST_AUTH_TOKEN - env: - CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} - CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} - CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} - CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} - NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} - PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }} - RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }} - SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} - SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }} - SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }} - SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }} - SCCACHE_DIST_REQUEST_TIMEOUT: ${{ env.get("SCCACHE_DIST_REQUEST_TIMEOUT", default="7140") }} - SCCACHE_DIST_SCHEDULER_URL: ${{ env.get("SCCACHE_DIST_SCHEDULER_URL", default="") }} - SCCACHE_ERROR_LOG: ${{ env.get("SCCACHE_ERROR_LOG", default="/tmp/sccache.log") }} - SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="0") }} - SCCACHE_NO_CACHE: ${{ env.get("SCCACHE_NO_CACHE", default="") }} - SCCACHE_RECACHE: ${{ env.get("SCCACHE_RECACHE", default="") }} - SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} - SCCACHE_S3_KEY_PREFIX: libcuvs/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} - SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }} - SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX: libcuvs/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}/conda/preprocessor-cache - SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE: ${{ env.get("SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE", default="true") }} - SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }} - SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }} - requirements: - build: - - ${{ compiler("c") }} - - ${{ compiler("cxx") }} - - ${{ compiler("cuda") }} - - cuda-version =${{ cuda_version }} - - cmake ${{ cmake_version }} - - ninja - - python - - if: cuda_major == "13" - then: - - cutile-python - - cuda-tileiras - - ${{ stdlib("c") }} - host: - - libnvjitlink-dev - - librmm =${{ minor_version }} - - libraft-headers =${{ minor_version }} - - libkvikio =${{ minor_version }} - - nccl ${{ nccl_version }} - - cuda-version =${{ cuda_version }} - - cuda-cudart-dev - - cuda-nvrtc-dev - - cuda-profiler-api - - libcublas-dev - - libcurand-dev - - libcusolver-dev - - libcusparse-dev - # These are used for bench-ann - - openblas - - if: linux64 - then: - - libaio - - libboost-devel =1.87 - - mkl-devel =2023 + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + - SCCACHE_DIST_AUTH_TOKEN + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} + PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }} + RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }} + RAPIDS_CUDA_VERSION: ${{ env.get("RAPIDS_CUDA_VERSION") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} + SCCACHE_CONF: ${{ env.get("SCCACHE_CONF", default="") }} + SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }} + SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }} + SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }} + SCCACHE_DIST_REQUEST_TIMEOUT: ${{ env.get("SCCACHE_DIST_REQUEST_TIMEOUT", default="7140") }} + SCCACHE_DIST_SCHEDULER_URL: ${{ env.get("SCCACHE_DIST_SCHEDULER_URL", default="") }} + SCCACHE_ERROR_LOG: ${{ env.get("SCCACHE_ERROR_LOG", default="/tmp/sccache.log") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="0") }} + SCCACHE_NO_CACHE: ${{ env.get("SCCACHE_NO_CACHE", default="") }} + SCCACHE_RECACHE: ${{ env.get("SCCACHE_RECACHE", default="") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} + SCCACHE_S3_KEY_PREFIX: libcuvs/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }} + SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX: libcuvs/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}/conda/preprocessor-cache + SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE: ${{ env.get("SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE", default="true") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }} + SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }} + requirements: + build: + - ${{ compiler("c") }} + - ${{ compiler("cxx") }} + - ${{ compiler("cuda") }} + - cuda-version =${{ cuda_version }} + - cmake ${{ cmake_version }} + - ninja + - python + - if: cuda_major == "13" + then: + - cutile-python + - cuda-tileiras + - ${{ stdlib("c") }} + host: + - libnvjitlink-dev + - librmm =${{ minor_version }} + - libraft-headers =${{ minor_version }} + - libkvikio =${{ minor_version }} + - nccl ${{ nccl_version }} + - cuda-version =${{ cuda_version }} + - cuda-cudart-dev + - cuda-nvrtc-dev + - cuda-profiler-api + - libcublas-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev + # These are used for bench-ann + - openblas + - if: linux64 + then: + - libaio + - libboost-devel =1.87 + - mkl-devel =2023 -outputs: - package: name: libcuvs-headers version: ${{ version }} + inherit: libcuvs-build build: string: cuda${{ cuda_major }}_${{ datetime_string }}_${{ head_rev }} dynamic_linking: @@ -163,6 +167,7 @@ outputs: - package: name: libcuvs version: ${{ version }} + inherit: libcuvs-build build: string: cuda${{ cuda_major }}_${{ datetime_string }}_${{ head_rev }} dynamic_linking: @@ -195,6 +200,8 @@ outputs: - libcusparse-dev - libnvjitlink-dev run: + - rapids-logger =0.3 + - libgomp - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - ${{ pin_subpackage("libcuvs-headers", exact=True) }} - libraft-headers =${{ minor_version }} @@ -228,6 +235,7 @@ outputs: - package: name: libcuvs-static version: ${{ version }} + inherit: libcuvs-build build: string: cuda${{ cuda_major }}_${{ datetime_string }}_${{ head_rev }} dynamic_linking: @@ -293,6 +301,7 @@ outputs: - package: name: libcuvs-tests version: ${{ version }} + inherit: libcuvs-build build: script: cmake --install cpp/build --component testing string: cuda${{ cuda_major }}_${{ datetime_string }}_${{ head_rev }} @@ -319,6 +328,11 @@ outputs: - libcusolver-dev - libcusparse-dev run: + - libucxx + - ucx + - libgomp + - librmm =${{ minor_version }} + - rapids-logger =0.3 - ${{ pin_subpackage("libcuvs-headers", exact=True) }} - ${{ pin_subpackage("libcuvs", exact=True) }} - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} @@ -350,6 +364,7 @@ outputs: - package: name: libcuvs-examples version: ${{ version }} + inherit: libcuvs-build build: string: cuda${{ cuda_major }}_${{ datetime_string }}_${{ head_rev }} dynamic_linking: @@ -380,6 +395,8 @@ outputs: - libcusparse-dev - libnvjitlink-dev run: + - rapids-logger =0.3 + - librmm =${{ minor_version }} - ${{ pin_subpackage("libcuvs-headers", exact=True) }} - ${{ pin_subpackage("libcuvs", exact=True) }} - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} @@ -411,6 +428,7 @@ outputs: - package: name: libcuvs-bench-ann version: ${{ version }} + inherit: libcuvs-build build: script: content: | @@ -421,6 +439,8 @@ outputs: # component associated with it. Probably a better way to do this. cp cpp/build/_deps/diskann-build/src/libdiskann.so $PREFIX/lib/ fi + env: + RAPIDS_CONDA_ARCH: ${{ env.get("RAPIDS_CONDA_ARCH") }} string: cuda${{ cuda_major }}_${{ datetime_string }}_${{ head_rev }} dynamic_linking: overlinking_behavior: "error" @@ -451,6 +471,11 @@ outputs: - libboost-devel =1.87 - mkl-devel =2023 run: + - if: aarch64 + then: libopenblas + - librmm =${{ minor_version }} + - rapids-logger =0.3 + - libgomp - ${{ pin_subpackage("libcuvs-headers", exact=True) }} - ${{ pin_subpackage("libcuvs", exact=True) }} - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} @@ -461,7 +486,9 @@ outputs: - libraft-headers =${{ minor_version }} - nccl - if: linux64 - then: mkl =2023 + then: + - mkl =2023 + - llvm-openmp ignore_run_exports: by_name: - cuda-cudart