From 63697792bd454de263ca9b60332806f1f24da631 Mon Sep 17 00:00:00 2001 From: Wesley Maxey Date: Fri, 16 Oct 2026 14:26:04 -0700 Subject: [PATCH 1/6] Run setup-cuda-compat on devcontainer/docker entry --- .devcontainer/cccl-entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/cccl-entrypoint.sh b/.devcontainer/cccl-entrypoint.sh index 490cf9a0fde3..273e423c1b10 100755 --- a/.devcontainer/cccl-entrypoint.sh +++ b/.devcontainer/cccl-entrypoint.sh @@ -8,6 +8,7 @@ if [[ -n "${GITHUB_ACTIONS:-}" ]]; then echo "::group::Initializing devcontainer..." fi +setup-cuda-compat; devcontainer-utils-post-create-command; devcontainer-utils-init-git; devcontainer-utils-post-attach-command; From 292be495d4dca29b577eeb1ffeadce320722d736 Mon Sep 17 00:00:00 2001 From: Wesley Maxey Date: Mon, 12 Oct 2026 14:13:39 -0700 Subject: [PATCH 2/6] CI matrix override --- ci/matrix.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/matrix.yaml b/ci/matrix.yaml index 49249427dbda..525cc1e7ef8c 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -79,6 +79,7 @@ workflows: # # IMPORTANT: Do NOT delete or remove the `override:` key below, even when it is empty. override: + - {jobs: ['test'], project: ['libcudacxx', 'cudax'], std: 'max', gpu: 'h100' } pull_request: # Old CTK: Oldest/newest supported host compilers: From c22829c358e56d59ad576cf36abd007c6917a9e6 Mon Sep 17 00:00:00 2001 From: Wesley Maxey Date: Mon, 12 Oct 2026 14:15:24 -0700 Subject: [PATCH 3/6] Force locality tests --- .../cuda/ccclrt/device/locality_domains.c2h.cu | 9 --------- 1 file changed, 9 deletions(-) diff --git a/libcudacxx/test/libcudacxx/cuda/ccclrt/device/locality_domains.c2h.cu b/libcudacxx/test/libcudacxx/cuda/ccclrt/device/locality_domains.c2h.cu index 39863aef35c5..707fc13f3d11 100644 --- a/libcudacxx/test/libcudacxx/cuda/ccclrt/device/locality_domains.c2h.cu +++ b/libcudacxx/test/libcudacxx/cuda/ccclrt/device/locality_domains.c2h.cu @@ -75,15 +75,6 @@ C2H_CCCLRT_TEST("locality domains", "[device][locality_domain]") return; } -# if _CCCL_CTK_AT_LEAST(13, 4) - // locality domains require CUDA 13.4 driver - if (test::cuda_driver_version() < 13040) - { - SUCCEED("Driver is too old for locality domain tests (requires CUDA 13.4 / R615 or later)"); - return; - } -# endif // _CCCL_CTK_AT_LEAST(13, 4) - SECTION("Returns at least one domain per device") { for (auto dev : cuda::devices) From f0630ff0c3f28654ec84dad8ddda16f7a3d062ff Mon Sep 17 00:00:00 2001 From: Wesley Maxey Date: Mon, 19 Oct 2026 11:02:59 -0700 Subject: [PATCH 4/6] Only enable forward compat if the GPU is NGC compatible --- .devcontainer/cccl-entrypoint.sh | 11 ++++++++- .../actions/workflow-build/build-workflow.py | 22 ++++++++++++++++- ci/matrix.yaml | 24 ++++++++++++------- .../ci/references/matrix_yaml.rst | 6 ++++- 4 files changed, 51 insertions(+), 12 deletions(-) diff --git a/.devcontainer/cccl-entrypoint.sh b/.devcontainer/cccl-entrypoint.sh index 273e423c1b10..3adf75263a15 100755 --- a/.devcontainer/cccl-entrypoint.sh +++ b/.devcontainer/cccl-entrypoint.sh @@ -8,7 +8,16 @@ if [[ -n "${GITHUB_ACTIONS:-}" ]]; then echo "::group::Initializing devcontainer..." fi -setup-cuda-compat; +# The CUDA forward compatibility (cuda-compat) libraries are only supported on +# data center GPUs; activating them on other GPUs breaks CUDA. GPU jobs set +# DATA_CENTER_GPU ('1' or '0') via the CI matrix, based on the runner's GPU. +# Local users with a data center GPU can opt in with: +# .devcontainer/launch.sh --env DATA_CENTER_GPU=1 +if [[ "${DATA_CENTER_GPU:-0}" = "1" ]]; then + setup-cuda-compat; +else + echo "DATA_CENTER_GPU is '${DATA_CENTER_GPU:-0}', skipping cuda-compat configuration."; +fi devcontainer-utils-post-create-command; devcontainer-utils-init-git; devcontainer-utils-post-attach-command; diff --git a/.github/actions/workflow-build/build-workflow.py b/.github/actions/workflow-build/build-workflow.py index ef52b2b0db39..e7b1531fad6b 100755 --- a/.github/actions/workflow-build/build-workflow.py +++ b/.github/actions/workflow-build/build-workflow.py @@ -281,6 +281,12 @@ def get_gpu(gpu_string): if "gpu_count" not in result: result["gpu_count"] = 1 + # dc (data center GPU) defaults to False. Used to decide whether the CUDA + # forward compatibility (cuda-compat) libraries may be activated in the + # container; they are only supported on data center GPUs. + if "dc" not in result: + result["dc"] = False + # runner_id defaults to the GPU key. Used in runner label templates as {gpu_id}. if "runner_id" not in result: result["runner_id"] = gpu_string @@ -597,7 +603,21 @@ def generate_dispatch_job_image(matrix_job, job_type): def generate_dispatch_job_environment(matrix_job, job_type): - return json.dumps(matrix_job.get("environment") or []) + environment = list(matrix_job.get("environment") or []) + + # Inject the GPU's data center classification into every GPU job. The + # devcontainer entrypoint uses this to decide whether to run the + # setup-cuda-compat script, since the CUDA forward compatibility + # (cuda-compat) libraries are only supported on data center GPUs. + # Explicitly emit both cases ('1' and '0') so mock jobs reproduce CI. + # Skip if the tag already set the variable to allow manual overrides. + if get_job_type_info(job_type)["gpu"] and not any( + env.startswith("DATA_CENTER_GPU=") for env in environment + ): + gpu = get_gpu(matrix_job["gpu"]) + environment.append(f"DATA_CENTER_GPU={'1' if gpu['dc'] else '0'}") + + return json.dumps(environment) def generate_dispatch_job_command(matrix_job, job_type): diff --git a/ci/matrix.yaml b/ci/matrix.yaml index 525cc1e7ef8c..12d14ff8d4ba 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -932,18 +932,24 @@ projects: # Defaults to the GPU key. Useful when the key includes a suffix # (e.g. 'h100_2gpu') but the runner label should use the base name # (e.g. 'h100'). +# dc -> Whether the GPU is a data center product. Injected into all GPU jobs as +# the DATA_CENTER_GPU environment variable ('1' or '0'), which the +# devcontainer entrypoint uses to decide whether to run the +# setup-cuda-compat script. The CUDA forward compatibility (cuda-compat) +# libraries are only supported on data center GPUs and must not be +# activated on other GPUs. gpus: - t4: { name: 'T4', sm: 75, runner: 't4-latest', gpu_count: 1 } # 16 GB, 10 runners - rtx2080: { name: 'RTX2080', sm: 75, runner: 'rtx2080-latest', gpu_count: 1 } # 8 GB, 12 runners - rtxa6000: { name: 'RTXA6000', sm: 86, runner: 'rtxa6000-latest', gpu_count: 1 } # 48 GB, 12 runners - l4: { name: 'L4', sm: 89, runner: 'l4-latest', gpu_count: 1 } # 24 GB, 48 runners - rtx4090: { name: 'RTX4090', sm: 89, runner: 'rtx4090-latest', gpu_count: 1 } # 24 GB, 10 runners - h100: { name: 'H100', sm: 90, runner: 'h100-latest', gpu_count: 1 } # 80 GB, 16 runners - h100_2gpu: { name: 'H100 2-GPU', sm: 90, runner: 'h100-latest', gpu_count: 2, runner_id: 'h100' } # 2 x 80 GB - gb300: { name: 'GB300', sm: 103, runner: 'gb300-latest', gpu_count: 1 } + t4: { name: 'T4', sm: 75, runner: 't4-latest', gpu_count: 1, dc: true } # 16 GB, 10 runners + rtx2080: { name: 'RTX2080', sm: 75, runner: 'rtx2080-latest', gpu_count: 1, dc: false } # 8 GB, 12 runners + rtxa6000: { name: 'RTXA6000', sm: 86, runner: 'rtxa6000-latest', gpu_count: 1, dc: true } # 48 GB, 12 runners + l4: { name: 'L4', sm: 89, runner: 'l4-latest', gpu_count: 1, dc: true } # 24 GB, 48 runners + rtx4090: { name: 'RTX4090', sm: 89, runner: 'rtx4090-latest', gpu_count: 1, dc: false } # 24 GB, 10 runners + h100: { name: 'H100', sm: 90, runner: 'h100-latest', gpu_count: 1, dc: true } # 80 GB, 16 runners + h100_2gpu: { name: 'H100 2-GPU', sm: 90, runner: 'h100-latest', gpu_count: 2, runner_id: 'h100', dc: true } # 2 x 80 GB + gb300: { name: 'GB300', sm: 103, runner: 'gb300-latest', gpu_count: 1, dc: true } # Very small number of runners on loan from cuda-python while we wait for our order to arrive. # Limit jobs on these: - rtxpro6000: { name: 'RTXPRO6000', sm: 120, runner: 'rtxpro6000-latest', gpu_count: 1 } + rtxpro6000: { name: 'RTXPRO6000', sm: 120, runner: 'rtxpro6000-latest', gpu_count: 1, dc: true } # Tags are used to define a `matrix job` in the workflow section. # diff --git a/docs/infrastructure/ci/references/matrix_yaml.rst b/docs/infrastructure/ci/references/matrix_yaml.rst index fc4d68825451..fb2375951b92 100644 --- a/docs/infrastructure/ci/references/matrix_yaml.rst +++ b/docs/infrastructure/ci/references/matrix_yaml.rst @@ -37,7 +37,7 @@ Top-level keys * - ``projects`` - Project definitions: supported standards, display name, and ``job_map`` expansions. * - ``gpus`` - - GPU runner pools and their ``sm`` value. + - GPU runner pools and their ``sm`` value and ``dc`` (data center GPU) classification. * - ``tags`` - The fields a job entry accepts, with ``required`` flags and defaults. @@ -119,6 +119,10 @@ Computed internally, not user-specified: - ``gpu`` requirement, ``cuda_ext``, ``name``, and ``invoke`` script details - also taken from the ``jobs`` section. - ``force_producer_ctk`` — set in the ``jobs`` section to pin a producer build's CTK independent of the consumer's ``ctk`` tag. Used mainly for python packaging special cases. +- ``DATA_CENTER_GPU`` — injected into the ``environment`` of every GPU job as ``1`` or ``0``, taken + from the selected GPU's ``dc`` field. The devcontainer entrypoint gates the CUDA forward + compatibility (``cuda-compat``) setup on this variable; forward compatibility is only + supported on data center GPUs. Annotated example ~~~~~~~~~~~~~~~~~ From 7b5e87cf052a9e6565571871fddafe9f7aaafc37 Mon Sep 17 00:00:00 2001 From: Wesley Maxey Date: Mon, 19 Oct 2026 13:32:27 -0700 Subject: [PATCH 5/6] Remove override --- ci/matrix.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/matrix.yaml b/ci/matrix.yaml index 12d14ff8d4ba..4844fe4f71ae 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -79,7 +79,6 @@ workflows: # # IMPORTANT: Do NOT delete or remove the `override:` key below, even when it is empty. override: - - {jobs: ['test'], project: ['libcudacxx', 'cudax'], std: 'max', gpu: 'h100' } pull_request: # Old CTK: Oldest/newest supported host compilers: From 2695204cb700325ff5906ecb3696b8887174d767 Mon Sep 17 00:00:00 2001 From: Wesley Maxey Date: Mon, 19 Oct 2026 13:32:46 -0700 Subject: [PATCH 6/6] Revert "Force locality tests" This reverts commit c22829c358e56d59ad576cf36abd007c6917a9e6. --- .../cuda/ccclrt/device/locality_domains.c2h.cu | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libcudacxx/test/libcudacxx/cuda/ccclrt/device/locality_domains.c2h.cu b/libcudacxx/test/libcudacxx/cuda/ccclrt/device/locality_domains.c2h.cu index 707fc13f3d11..39863aef35c5 100644 --- a/libcudacxx/test/libcudacxx/cuda/ccclrt/device/locality_domains.c2h.cu +++ b/libcudacxx/test/libcudacxx/cuda/ccclrt/device/locality_domains.c2h.cu @@ -75,6 +75,15 @@ C2H_CCCLRT_TEST("locality domains", "[device][locality_domain]") return; } +# if _CCCL_CTK_AT_LEAST(13, 4) + // locality domains require CUDA 13.4 driver + if (test::cuda_driver_version() < 13040) + { + SUCCEED("Driver is too old for locality domain tests (requires CUDA 13.4 / R615 or later)"); + return; + } +# endif // _CCCL_CTK_AT_LEAST(13, 4) + SECTION("Returns at least one domain per device") { for (auto dev : cuda::devices)