diff --git a/.devcontainer/cccl-entrypoint.sh b/.devcontainer/cccl-entrypoint.sh index 490cf9a0fde3..3adf75263a15 100755 --- a/.devcontainer/cccl-entrypoint.sh +++ b/.devcontainer/cccl-entrypoint.sh @@ -8,6 +8,16 @@ if [[ -n "${GITHUB_ACTIONS:-}" ]]; then echo "::group::Initializing devcontainer..." fi +# The CUDA forward compatibility (cuda-compat) libraries are only supported on +# data center GPUs; activating them on other GPUs breaks CUDA. GPU jobs set +# DATA_CENTER_GPU ('1' or '0') via the CI matrix, based on the runner's GPU. +# Local users with a data center GPU can opt in with: +# .devcontainer/launch.sh --env DATA_CENTER_GPU=1 +if [[ "${DATA_CENTER_GPU:-0}" = "1" ]]; then + setup-cuda-compat; +else + echo "DATA_CENTER_GPU is '${DATA_CENTER_GPU:-0}', skipping cuda-compat configuration."; +fi devcontainer-utils-post-create-command; devcontainer-utils-init-git; devcontainer-utils-post-attach-command; diff --git a/.github/actions/workflow-build/build-workflow.py b/.github/actions/workflow-build/build-workflow.py index ef52b2b0db39..e7b1531fad6b 100755 --- a/.github/actions/workflow-build/build-workflow.py +++ b/.github/actions/workflow-build/build-workflow.py @@ -281,6 +281,12 @@ def get_gpu(gpu_string): if "gpu_count" not in result: result["gpu_count"] = 1 + # dc (data center GPU) defaults to False. Used to decide whether the CUDA + # forward compatibility (cuda-compat) libraries may be activated in the + # container; they are only supported on data center GPUs. + if "dc" not in result: + result["dc"] = False + # runner_id defaults to the GPU key. Used in runner label templates as {gpu_id}. if "runner_id" not in result: result["runner_id"] = gpu_string @@ -597,7 +603,21 @@ def generate_dispatch_job_image(matrix_job, job_type): def generate_dispatch_job_environment(matrix_job, job_type): - return json.dumps(matrix_job.get("environment") or []) + environment = list(matrix_job.get("environment") or []) + + # Inject the GPU's data center classification into every GPU job. The + # devcontainer entrypoint uses this to decide whether to run the + # setup-cuda-compat script, since the CUDA forward compatibility + # (cuda-compat) libraries are only supported on data center GPUs. + # Explicitly emit both cases ('1' and '0') so mock jobs reproduce CI. + # Skip if the tag already set the variable to allow manual overrides. + if get_job_type_info(job_type)["gpu"] and not any( + env.startswith("DATA_CENTER_GPU=") for env in environment + ): + gpu = get_gpu(matrix_job["gpu"]) + environment.append(f"DATA_CENTER_GPU={'1' if gpu['dc'] else '0'}") + + return json.dumps(environment) def generate_dispatch_job_command(matrix_job, job_type): diff --git a/ci/matrix.yaml b/ci/matrix.yaml index 49249427dbda..4844fe4f71ae 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -931,18 +931,24 @@ projects: # Defaults to the GPU key. Useful when the key includes a suffix # (e.g. 'h100_2gpu') but the runner label should use the base name # (e.g. 'h100'). +# dc -> Whether the GPU is a data center product. Injected into all GPU jobs as +# the DATA_CENTER_GPU environment variable ('1' or '0'), which the +# devcontainer entrypoint uses to decide whether to run the +# setup-cuda-compat script. The CUDA forward compatibility (cuda-compat) +# libraries are only supported on data center GPUs and must not be +# activated on other GPUs. gpus: - t4: { name: 'T4', sm: 75, runner: 't4-latest', gpu_count: 1 } # 16 GB, 10 runners - rtx2080: { name: 'RTX2080', sm: 75, runner: 'rtx2080-latest', gpu_count: 1 } # 8 GB, 12 runners - rtxa6000: { name: 'RTXA6000', sm: 86, runner: 'rtxa6000-latest', gpu_count: 1 } # 48 GB, 12 runners - l4: { name: 'L4', sm: 89, runner: 'l4-latest', gpu_count: 1 } # 24 GB, 48 runners - rtx4090: { name: 'RTX4090', sm: 89, runner: 'rtx4090-latest', gpu_count: 1 } # 24 GB, 10 runners - h100: { name: 'H100', sm: 90, runner: 'h100-latest', gpu_count: 1 } # 80 GB, 16 runners - h100_2gpu: { name: 'H100 2-GPU', sm: 90, runner: 'h100-latest', gpu_count: 2, runner_id: 'h100' } # 2 x 80 GB - gb300: { name: 'GB300', sm: 103, runner: 'gb300-latest', gpu_count: 1 } + t4: { name: 'T4', sm: 75, runner: 't4-latest', gpu_count: 1, dc: true } # 16 GB, 10 runners + rtx2080: { name: 'RTX2080', sm: 75, runner: 'rtx2080-latest', gpu_count: 1, dc: false } # 8 GB, 12 runners + rtxa6000: { name: 'RTXA6000', sm: 86, runner: 'rtxa6000-latest', gpu_count: 1, dc: true } # 48 GB, 12 runners + l4: { name: 'L4', sm: 89, runner: 'l4-latest', gpu_count: 1, dc: true } # 24 GB, 48 runners + rtx4090: { name: 'RTX4090', sm: 89, runner: 'rtx4090-latest', gpu_count: 1, dc: false } # 24 GB, 10 runners + h100: { name: 'H100', sm: 90, runner: 'h100-latest', gpu_count: 1, dc: true } # 80 GB, 16 runners + h100_2gpu: { name: 'H100 2-GPU', sm: 90, runner: 'h100-latest', gpu_count: 2, runner_id: 'h100', dc: true } # 2 x 80 GB + gb300: { name: 'GB300', sm: 103, runner: 'gb300-latest', gpu_count: 1, dc: true } # Very small number of runners on loan from cuda-python while we wait for our order to arrive. # Limit jobs on these: - rtxpro6000: { name: 'RTXPRO6000', sm: 120, runner: 'rtxpro6000-latest', gpu_count: 1 } + rtxpro6000: { name: 'RTXPRO6000', sm: 120, runner: 'rtxpro6000-latest', gpu_count: 1, dc: true } # Tags are used to define a `matrix job` in the workflow section. # diff --git a/docs/infrastructure/ci/references/matrix_yaml.rst b/docs/infrastructure/ci/references/matrix_yaml.rst index fc4d68825451..fb2375951b92 100644 --- a/docs/infrastructure/ci/references/matrix_yaml.rst +++ b/docs/infrastructure/ci/references/matrix_yaml.rst @@ -37,7 +37,7 @@ Top-level keys * - ``projects`` - Project definitions: supported standards, display name, and ``job_map`` expansions. * - ``gpus`` - - GPU runner pools and their ``sm`` value. + - GPU runner pools and their ``sm`` value and ``dc`` (data center GPU) classification. * - ``tags`` - The fields a job entry accepts, with ``required`` flags and defaults. @@ -119,6 +119,10 @@ Computed internally, not user-specified: - ``gpu`` requirement, ``cuda_ext``, ``name``, and ``invoke`` script details - also taken from the ``jobs`` section. - ``force_producer_ctk`` — set in the ``jobs`` section to pin a producer build's CTK independent of the consumer's ``ctk`` tag. Used mainly for python packaging special cases. +- ``DATA_CENTER_GPU`` — injected into the ``environment`` of every GPU job as ``1`` or ``0``, taken + from the selected GPU's ``dc`` field. The devcontainer entrypoint gates the CUDA forward + compatibility (``cuda-compat``) setup on this variable; forward compatibility is only + supported on data center GPUs. Annotated example ~~~~~~~~~~~~~~~~~