From d51f605a68eedec7b75119d2146bcf7dd577b344 Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 02:51:46 +0800
Subject: [PATCH 01/14] [CI] Upgrade H-Coverage to Python 3.12

Co-authored-by: Codex <noreply@openai.com>
---
 .github/workflows/H-Coverage.yml | 50 ++++++++++++++++++++++++--------
 1 file changed, 38 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/H-Coverage.yml b/.github/workflows/H-Coverage.yml
index 49f9efa3ba31fc..cd8e8ba3f3fb2c 100644
--- a/.github/workflows/H-Coverage.yml
+++ b/.github/workflows/H-Coverage.yml
@@ -104,7 +104,7 @@ jobs:
           LITE_GIT_TAG: develop
           WITH_UNITY_BUILD: "ON"
           WITH_FA_BUILD_WITH_CACHE: "ON"
-          PY_VERSION: "3.10"
+          PY_VERSION: "3.12"
           INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
           CCACHE_MAXSIZE: 50G
           CCACHE_LIMIT_MULTIPLE: 0.8
@@ -191,13 +191,17 @@ jobs:
           mkdir -p ${CFS_DIR}/.cache/coverage
           mkdir -p ${CFS_DIR}/.ccache/coverage
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-12.9/compat
+          python_bin=$(command -v python${PY_VERSION}) || exit 1
+          pip_bin=$(command -v pip${PY_VERSION}) || exit 1
+          ln -sf ${python_bin} /usr/local/bin/python
+          ln -sf ${pip_bin} /usr/local/bin/pip
           source ${{ github.workspace }}/../../../proxy
           bash ${ci_scripts}/cmake-predownload.sh
           bash ${ci_scripts}/cmake_preload_data_file.sh
           pip install -r python/requirements.txt
           mkdir -p build && cd build
           ccache -z
-          cmake .. -DPY_VERSION=3.10 -DWITH_GPU=ON -DWITH_DISTRIBUTE=ON -DWITH_TESTING=ON -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN="90" -DFA_JOB_POOLS_COMPILE=1 -DWITH_CUDNN_FRONTEND=ON -DON_INFER=OFF -DWITH_NVSHMEM=ON
+          cmake .. -DPY_VERSION=3.12 -DWITH_GPU=ON -DWITH_DISTRIBUTE=ON -DWITH_TESTING=ON -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN="90" -DFA_JOB_POOLS_COMPILE=1 -DWITH_CUDNN_FRONTEND=ON -DON_INFER=OFF -DWITH_NVSHMEM=ON
           make -j20
           EXIT_CODE=$?
           ccache -s
@@ -223,7 +227,7 @@ jobs:
         env:
           home_path: ${{ github.workspace }}/..
           bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
-          paddle_whl: paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
+          paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           echo "::group::Install bce-python-sdk"
@@ -238,7 +242,7 @@ jobs:
           fi
           cd /paddle
           mv /Paddle.tar.gz .
-          cp ./build/python/dist/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl .
+          cp ./build/python/dist/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl .
           echo "Uploading Paddle.tar.gz"
           python ${{ env.bos_file }} Paddle.tar.gz paddle-github-action/PR/h-coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
           echo "Uploading coverage wheel"
@@ -283,7 +287,7 @@ jobs:
           PADDLE_VERSION: 0.0.0
           WITH_DISTRIBUTE: "ON"
           WITH_UNITY_BUILD: "ON"
-          PY_VERSION: "3.10"
+          PY_VERSION: "3.12"
           WITH_SHARED_PHI: "ON"
           GPU_DEVICES: ${{ env.GPU_DEVICES }}
           WITH_CINN: "ON"
@@ -359,6 +363,10 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           source ${{ github.workspace }}/../../../proxy
+          python_bin=$(command -v python${PY_VERSION}) || exit 1
+          pip_bin=$(command -v pip${PY_VERSION}) || exit 1
+          ln -sf ${python_bin} /usr/local/bin/python
+          ln -sf ${pip_bin} /usr/local/bin/pip
           pip install build//python/dist/*.whl --no-deps
           pip install -r python/unittest_py/requirements.txt
           bash $ci_scripts/h-test.sh
@@ -392,6 +400,7 @@ jobs:
     env:
       TASK: fleet-ci-paddle-build-whl-${{ github.event.pull_request.number }}
       docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-dev"
+      PY_VERSION: "3.12"
     steps:
       - name: Check docker image and run container
         env:
@@ -425,6 +434,7 @@ jobs:
             -e CCACHE_LIMIT_MULTIPLE \
             -e CCACHE_STATSLOG \
             -e CCACHE_SLOPPINESS \
+            -e PY_VERSION \
             -w /paddle --network host ${docker_image}
 
       - name: uv build whl
@@ -433,6 +443,10 @@ jobs:
           rm -rf * .[^.]*
           set -x
           source ${{ github.workspace }}/../../../proxy
+          python_bin=$(command -v python${PY_VERSION}) || exit 1
+          pip_bin=$(command -v pip${PY_VERSION}) || exit 1
+          ln -sf ${python_bin} /usr/local/bin/python
+          ln -sf ${pip_bin} /usr/local/bin/pip
           apt-get update
           apt-get install -y libnvidia-compute-535
           python -m pip install --upgrade pip wheel setuptools
@@ -458,9 +472,9 @@ jobs:
           export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:$LD_LIBRARY_PATH
           export IS_NVIDIA=True
           echo "Downloading Paddle.tar.gz from cfs"
-          wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl --no-check-certificate
+          wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
           pip uninstall paddlepaddle-gpu -y
-          pip install paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
+          pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
           pip install paddle-nvidia-nvshmem-cu12 --extra-index-url https://www.paddlepaddle.org.cn/packages/nightly/cu129/
           uv build --wheel --no-build-isolation -v --python $(which python)
           '
@@ -469,7 +483,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -ce '
           set -x
-          OPS_DST=paddlefleet_ops-0.0.0-cp310-cp310-linux_x86_64.whl
+          OPS_DST=paddlefleet_ops-0.0.0-cp312-cp312-linux_x86_64.whl
           wget -q --no-proxy "https://paddle-whl.bj.bcebos.com/nightly/cu129/paddlefleet-ops/${OPS_DST}" -O dist/${OPS_DST}
           '
 
@@ -506,6 +520,7 @@ jobs:
       PIP_CACHE_DIR: /root/.cache/pip
       CACHE_DIR: /root/.cache
       TASK: paddle-fleet-CI-${{ github.event.pull_request.number }}-single-card-test
+      PY_VERSION: "3.12"
     steps:
       - name: Determine the runner
         run: |
@@ -535,6 +550,7 @@ jobs:
             -e no_proxy \
             -e CI_name \
             -e PIP_CACHE_DIR \
+            -e PY_VERSION \
             -e work_dir \
             -e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
             -e GITHUB_HEAD_REF="${{ github.head_ref }}" \
@@ -550,6 +566,10 @@ jobs:
           docker exec -t ${{ env.container_name }} /bin/bash -ce '
           rm -rf * .[^.]*
           source /root/proxy
+          python_bin=$(command -v python${PY_VERSION}) || exit 1
+          pip_bin=$(command -v pip${PY_VERSION}) || exit 1
+          ln -sf ${python_bin} /usr/local/bin/python
+          ln -sf ${pip_bin} /usr/local/bin/pip
           export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
           wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
@@ -570,12 +590,12 @@ jobs:
           mkdir -p /PaddlePaddle
           cd /PaddlePaddle
           echo "Downloading Paddle.tar.gz from cfs"
-          wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl --no-check-certificate
+          wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
           export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
           export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
           export UV_HTTP_TIMEOUT=300
           pip uninstall paddlepaddle-gpu -y
-          pip install paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
+          pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
           echo "paddlefleet commit:"
           python -c "import paddlefleet; print(paddlefleet.version.commit)"
           '
@@ -625,6 +645,7 @@ jobs:
       PIP_CACHE_DIR: /root/.cache/pip
       TASK: paddle-fleet-CI-${{ github.event.pull_request.number }}-multi-card_test
       docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-dev"
+      PY_VERSION: "3.12"
     steps:
       - name: Check docker image and run container
         run: |
@@ -647,6 +668,7 @@ jobs:
             -e no_proxy \
             -e CI_name \
             -e PIP_CACHE_DIR \
+            -e PY_VERSION \
             -e work_dir \
             -e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
             -e GITHUB_HEAD_REF="${{ github.head_ref }}" \
@@ -663,6 +685,10 @@ jobs:
           docker exec -t ${{ env.container_name }} /bin/bash -ce '
           rm -rf * .[^.]*
           source /root/proxy
+          python_bin=$(command -v python${PY_VERSION}) || exit 1
+          pip_bin=$(command -v pip${PY_VERSION}) || exit 1
+          ln -sf ${python_bin} /usr/local/bin/python
+          ln -sf ${pip_bin} /usr/local/bin/pip
           export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
           wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
@@ -683,12 +709,12 @@ jobs:
           mkdir -p /PaddlePaddle
           cd /PaddlePaddle
           echo "Downloading Paddle.tar.gz from cfs"
-          wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl --no-check-certificate
+          wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
           export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
           export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
           export UV_HTTP_TIMEOUT=300
           pip uninstall paddlepaddle-gpu -y
-          pip install paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
+          pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
           echo "paddlefleet commit:"
           python -c "import paddlefleet; print(paddlefleet.version.commit)"
           '

From e3afaf133351c9f27ad1ec3b8e55cda6004236f0 Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 03:17:54 +0800
Subject: [PATCH 02/14] [CI] Provision Python 3.12 in H-Coverage containers

Co-authored-by: Codex <noreply@openai.com>
---
 .github/workflows/H-Coverage.yml | 277 ++++++++++++++++++++++++++++---
 1 file changed, 256 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/H-Coverage.yml b/.github/workflows/H-Coverage.yml
index cd8e8ba3f3fb2c..049c464b1df8ef 100644
--- a/.github/workflows/H-Coverage.yml
+++ b/.github/workflows/H-Coverage.yml
@@ -179,6 +179,7 @@ jobs:
       - name: Build
         run: |
           docker exec -t ${{ env.container_name }}  /bin/bash -c '
+          source ${{ github.workspace }}/../../../proxy
           flashattn_version=$(git submodule status|grep flashattn|awk "{print \$1}"|sed "s#-##g")
           echo flashattn_version:$flashattn_version
           url="https://xly-devops.bj.bcebos.com/gpups/flash-attention/cu90/flashattn_libs_${flashattn_version}.tar"
@@ -191,11 +192,57 @@ jobs:
           mkdir -p ${CFS_DIR}/.cache/coverage
           mkdir -p ${CFS_DIR}/.ccache/coverage
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-12.9/compat
-          python_bin=$(command -v python${PY_VERSION}) || exit 1
-          pip_bin=$(command -v pip${PY_VERSION}) || exit 1
-          ln -sf ${python_bin} /usr/local/bin/python
-          ln -sf ${pip_bin} /usr/local/bin/pip
-          source ${{ github.workspace }}/../../../proxy
+          setup_target_python() {
+            local target_python="python${PY_VERSION}"
+            local target_pip="pip${PY_VERSION}"
+            local opt_python_dir="/opt/_internal/cpython-${PY_VERSION}.0/bin"
+            local apt_updated=0
+            apt_update_once() {
+              if [ "${apt_updated}" != "1" ]; then
+                apt-get update
+                apt_updated=1
+              fi
+            }
+            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
+              if [ -d "${opt_python_dir}" ]; then
+                export PATH="${opt_python_dir}:${PATH}"
+              fi
+            fi
+            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
+              apt_update_once
+              if ! apt-cache show "${target_python}" >/dev/null 2>&1; then
+                apt-get install -y software-properties-common ca-certificates curl
+                add-apt-repository -y ppa:deadsnakes/ppa
+                apt_updated=0
+                apt_update_once
+              fi
+              apt-get install -y "${target_python}" "${target_python}-dev" ca-certificates curl
+            fi
+            if ! command -v "${target_pip}" >/dev/null 2>&1; then
+              if ! command -v curl >/dev/null 2>&1; then
+                apt_update_once
+                apt-get install -y ca-certificates curl
+              fi
+              curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
+              "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
+            fi
+            python_bin=$(command -v "${target_python}" || true)
+            pip_bin=$(command -v "${target_pip}" || true)
+            if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
+              echo "ERROR: Python ${PY_VERSION} or pip ${PY_VERSION} is unavailable after setup." >&2
+              echo "PATH=${PATH}" >&2
+              command -v "${target_python}" || true
+              command -v "${target_pip}" || true
+              exit 1
+            fi
+            ln -sf "${python_bin}" /usr/local/bin/python
+            ln -sf "${pip_bin}" /usr/local/bin/pip
+            echo "Using python: $(command -v python)"
+            python --version
+            echo "Using pip: $(command -v pip)"
+            pip --version
+          }
+          setup_target_python
           bash ${ci_scripts}/cmake-predownload.sh
           bash ${ci_scripts}/cmake_preload_data_file.sh
           pip install -r python/requirements.txt
@@ -363,10 +410,57 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           source ${{ github.workspace }}/../../../proxy
-          python_bin=$(command -v python${PY_VERSION}) || exit 1
-          pip_bin=$(command -v pip${PY_VERSION}) || exit 1
-          ln -sf ${python_bin} /usr/local/bin/python
-          ln -sf ${pip_bin} /usr/local/bin/pip
+          setup_target_python() {
+            local target_python="python${PY_VERSION}"
+            local target_pip="pip${PY_VERSION}"
+            local opt_python_dir="/opt/_internal/cpython-${PY_VERSION}.0/bin"
+            local apt_updated=0
+            apt_update_once() {
+              if [ "${apt_updated}" != "1" ]; then
+                apt-get update
+                apt_updated=1
+              fi
+            }
+            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
+              if [ -d "${opt_python_dir}" ]; then
+                export PATH="${opt_python_dir}:${PATH}"
+              fi
+            fi
+            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
+              apt_update_once
+              if ! apt-cache show "${target_python}" >/dev/null 2>&1; then
+                apt-get install -y software-properties-common ca-certificates curl
+                add-apt-repository -y ppa:deadsnakes/ppa
+                apt_updated=0
+                apt_update_once
+              fi
+              apt-get install -y "${target_python}" "${target_python}-dev" ca-certificates curl
+            fi
+            if ! command -v "${target_pip}" >/dev/null 2>&1; then
+              if ! command -v curl >/dev/null 2>&1; then
+                apt_update_once
+                apt-get install -y ca-certificates curl
+              fi
+              curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
+              "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
+            fi
+            python_bin=$(command -v "${target_python}" || true)
+            pip_bin=$(command -v "${target_pip}" || true)
+            if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
+              echo "ERROR: Python ${PY_VERSION} or pip ${PY_VERSION} is unavailable after setup." >&2
+              echo "PATH=${PATH}" >&2
+              command -v "${target_python}" || true
+              command -v "${target_pip}" || true
+              exit 1
+            fi
+            ln -sf "${python_bin}" /usr/local/bin/python
+            ln -sf "${pip_bin}" /usr/local/bin/pip
+            echo "Using python: $(command -v python)"
+            python --version
+            echo "Using pip: $(command -v pip)"
+            pip --version
+          }
+          setup_target_python
           pip install build//python/dist/*.whl --no-deps
           pip install -r python/unittest_py/requirements.txt
           bash $ci_scripts/h-test.sh
@@ -443,10 +537,57 @@ jobs:
           rm -rf * .[^.]*
           set -x
           source ${{ github.workspace }}/../../../proxy
-          python_bin=$(command -v python${PY_VERSION}) || exit 1
-          pip_bin=$(command -v pip${PY_VERSION}) || exit 1
-          ln -sf ${python_bin} /usr/local/bin/python
-          ln -sf ${pip_bin} /usr/local/bin/pip
+          setup_target_python() {
+            local target_python="python${PY_VERSION}"
+            local target_pip="pip${PY_VERSION}"
+            local opt_python_dir="/opt/_internal/cpython-${PY_VERSION}.0/bin"
+            local apt_updated=0
+            apt_update_once() {
+              if [ "${apt_updated}" != "1" ]; then
+                apt-get update
+                apt_updated=1
+              fi
+            }
+            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
+              if [ -d "${opt_python_dir}" ]; then
+                export PATH="${opt_python_dir}:${PATH}"
+              fi
+            fi
+            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
+              apt_update_once
+              if ! apt-cache show "${target_python}" >/dev/null 2>&1; then
+                apt-get install -y software-properties-common ca-certificates curl
+                add-apt-repository -y ppa:deadsnakes/ppa
+                apt_updated=0
+                apt_update_once
+              fi
+              apt-get install -y "${target_python}" "${target_python}-dev" ca-certificates curl
+            fi
+            if ! command -v "${target_pip}" >/dev/null 2>&1; then
+              if ! command -v curl >/dev/null 2>&1; then
+                apt_update_once
+                apt-get install -y ca-certificates curl
+              fi
+              curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
+              "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
+            fi
+            python_bin=$(command -v "${target_python}" || true)
+            pip_bin=$(command -v "${target_pip}" || true)
+            if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
+              echo "ERROR: Python ${PY_VERSION} or pip ${PY_VERSION} is unavailable after setup." >&2
+              echo "PATH=${PATH}" >&2
+              command -v "${target_python}" || true
+              command -v "${target_pip}" || true
+              exit 1
+            fi
+            ln -sf "${python_bin}" /usr/local/bin/python
+            ln -sf "${pip_bin}" /usr/local/bin/pip
+            echo "Using python: $(command -v python)"
+            python --version
+            echo "Using pip: $(command -v pip)"
+            pip --version
+          }
+          setup_target_python
           apt-get update
           apt-get install -y libnvidia-compute-535
           python -m pip install --upgrade pip wheel setuptools
@@ -566,10 +707,57 @@ jobs:
           docker exec -t ${{ env.container_name }} /bin/bash -ce '
           rm -rf * .[^.]*
           source /root/proxy
-          python_bin=$(command -v python${PY_VERSION}) || exit 1
-          pip_bin=$(command -v pip${PY_VERSION}) || exit 1
-          ln -sf ${python_bin} /usr/local/bin/python
-          ln -sf ${pip_bin} /usr/local/bin/pip
+          setup_target_python() {
+            local target_python="python${PY_VERSION}"
+            local target_pip="pip${PY_VERSION}"
+            local opt_python_dir="/opt/_internal/cpython-${PY_VERSION}.0/bin"
+            local apt_updated=0
+            apt_update_once() {
+              if [ "${apt_updated}" != "1" ]; then
+                apt-get update
+                apt_updated=1
+              fi
+            }
+            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
+              if [ -d "${opt_python_dir}" ]; then
+                export PATH="${opt_python_dir}:${PATH}"
+              fi
+            fi
+            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
+              apt_update_once
+              if ! apt-cache show "${target_python}" >/dev/null 2>&1; then
+                apt-get install -y software-properties-common ca-certificates curl
+                add-apt-repository -y ppa:deadsnakes/ppa
+                apt_updated=0
+                apt_update_once
+              fi
+              apt-get install -y "${target_python}" "${target_python}-dev" ca-certificates curl
+            fi
+            if ! command -v "${target_pip}" >/dev/null 2>&1; then
+              if ! command -v curl >/dev/null 2>&1; then
+                apt_update_once
+                apt-get install -y ca-certificates curl
+              fi
+              curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
+              "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
+            fi
+            python_bin=$(command -v "${target_python}" || true)
+            pip_bin=$(command -v "${target_pip}" || true)
+            if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
+              echo "ERROR: Python ${PY_VERSION} or pip ${PY_VERSION} is unavailable after setup." >&2
+              echo "PATH=${PATH}" >&2
+              command -v "${target_python}" || true
+              command -v "${target_pip}" || true
+              exit 1
+            fi
+            ln -sf "${python_bin}" /usr/local/bin/python
+            ln -sf "${pip_bin}" /usr/local/bin/pip
+            echo "Using python: $(command -v python)"
+            python --version
+            echo "Using pip: $(command -v pip)"
+            pip --version
+          }
+          setup_target_python
           export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
           wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
@@ -685,10 +873,57 @@ jobs:
           docker exec -t ${{ env.container_name }} /bin/bash -ce '
           rm -rf * .[^.]*
           source /root/proxy
-          python_bin=$(command -v python${PY_VERSION}) || exit 1
-          pip_bin=$(command -v pip${PY_VERSION}) || exit 1
-          ln -sf ${python_bin} /usr/local/bin/python
-          ln -sf ${pip_bin} /usr/local/bin/pip
+          setup_target_python() {
+            local target_python="python${PY_VERSION}"
+            local target_pip="pip${PY_VERSION}"
+            local opt_python_dir="/opt/_internal/cpython-${PY_VERSION}.0/bin"
+            local apt_updated=0
+            apt_update_once() {
+              if [ "${apt_updated}" != "1" ]; then
+                apt-get update
+                apt_updated=1
+              fi
+            }
+            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
+              if [ -d "${opt_python_dir}" ]; then
+                export PATH="${opt_python_dir}:${PATH}"
+              fi
+            fi
+            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
+              apt_update_once
+              if ! apt-cache show "${target_python}" >/dev/null 2>&1; then
+                apt-get install -y software-properties-common ca-certificates curl
+                add-apt-repository -y ppa:deadsnakes/ppa
+                apt_updated=0
+                apt_update_once
+              fi
+              apt-get install -y "${target_python}" "${target_python}-dev" ca-certificates curl
+            fi
+            if ! command -v "${target_pip}" >/dev/null 2>&1; then
+              if ! command -v curl >/dev/null 2>&1; then
+                apt_update_once
+                apt-get install -y ca-certificates curl
+              fi
+              curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
+              "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
+            fi
+            python_bin=$(command -v "${target_python}" || true)
+            pip_bin=$(command -v "${target_pip}" || true)
+            if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
+              echo "ERROR: Python ${PY_VERSION} or pip ${PY_VERSION} is unavailable after setup." >&2
+              echo "PATH=${PATH}" >&2
+              command -v "${target_python}" || true
+              command -v "${target_pip}" || true
+              exit 1
+            fi
+            ln -sf "${python_bin}" /usr/local/bin/python
+            ln -sf "${pip_bin}" /usr/local/bin/pip
+            echo "Using python: $(command -v python)"
+            python --version
+            echo "Using pip: $(command -v pip)"
+            pip --version
+          }
+          setup_target_python
           export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
           wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz

From 00e0bf14b3a8400fd8643ac610890633b24e5ee9 Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 03:59:10 +0800
Subject: [PATCH 03/14] [CI] Install setuptools for Python 3.12 H-Coverage
 wheel

Co-authored-by: Codex <noreply@openai.com>
---
 .github/workflows/H-Coverage.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/H-Coverage.yml b/.github/workflows/H-Coverage.yml
index 049c464b1df8ef..8247f2ab366760 100644
--- a/.github/workflows/H-Coverage.yml
+++ b/.github/workflows/H-Coverage.yml
@@ -226,6 +226,8 @@ jobs:
               curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
               "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
             fi
+            "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
+            export SETUPTOOLS_USE_DISTUTILS=local
             python_bin=$(command -v "${target_python}" || true)
             pip_bin=$(command -v "${target_pip}" || true)
             if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
@@ -444,6 +446,8 @@ jobs:
               curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
               "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
             fi
+            "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
+            export SETUPTOOLS_USE_DISTUTILS=local
             python_bin=$(command -v "${target_python}" || true)
             pip_bin=$(command -v "${target_pip}" || true)
             if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
@@ -571,6 +575,8 @@ jobs:
               curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
               "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
             fi
+            "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
+            export SETUPTOOLS_USE_DISTUTILS=local
             python_bin=$(command -v "${target_python}" || true)
             pip_bin=$(command -v "${target_pip}" || true)
             if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
@@ -741,6 +747,8 @@ jobs:
               curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
               "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
             fi
+            "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
+            export SETUPTOOLS_USE_DISTUTILS=local
             python_bin=$(command -v "${target_python}" || true)
             pip_bin=$(command -v "${target_pip}" || true)
             if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
@@ -907,6 +915,8 @@ jobs:
               curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
               "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
             fi
+            "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
+            export SETUPTOOLS_USE_DISTUTILS=local
             python_bin=$(command -v "${target_python}" || true)
             pip_bin=$(command -v "${target_pip}" || true)
             if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then

From 2c569991e055a21be005ecb425b6d6ff551b8fb3 Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 04:41:30 +0800
Subject: [PATCH 04/14] [CI] Allow Python 3.12 pip installs in H-Coverage Fleet

Co-authored-by: Codex <noreply@openai.com>
---
 .github/workflows/H-Coverage.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/H-Coverage.yml b/.github/workflows/H-Coverage.yml
index 8247f2ab366760..4622946427d67b 100644
--- a/.github/workflows/H-Coverage.yml
+++ b/.github/workflows/H-Coverage.yml
@@ -226,6 +226,7 @@ jobs:
               curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
               "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
             fi
+            "${target_python}" -m pip config set global.break-system-packages true || true
             "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
             export SETUPTOOLS_USE_DISTUTILS=local
             python_bin=$(command -v "${target_python}" || true)
@@ -446,6 +447,7 @@ jobs:
               curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
               "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
             fi
+            "${target_python}" -m pip config set global.break-system-packages true || true
             "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
             export SETUPTOOLS_USE_DISTUTILS=local
             python_bin=$(command -v "${target_python}" || true)
@@ -575,6 +577,7 @@ jobs:
               curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
               "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
             fi
+            "${target_python}" -m pip config set global.break-system-packages true || true
             "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
             export SETUPTOOLS_USE_DISTUTILS=local
             python_bin=$(command -v "${target_python}" || true)
@@ -596,7 +599,7 @@ jobs:
           setup_target_python
           apt-get update
           apt-get install -y libnvidia-compute-535
-          python -m pip install --upgrade pip wheel setuptools
+          python -m pip install --upgrade pip wheel "setuptools==68.2.0" || python -m pip install --upgrade pip wheel "setuptools==68.2.0" --break-system-packages
           git clone https://github.com/PaddlePaddle/PaddleFleet.git .
           git config --global --add safe.directory /paddle
           git config user.name "PaddleCI"
@@ -747,6 +750,7 @@ jobs:
               curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
               "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
             fi
+            "${target_python}" -m pip config set global.break-system-packages true || true
             "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
             export SETUPTOOLS_USE_DISTUTILS=local
             python_bin=$(command -v "${target_python}" || true)
@@ -915,6 +919,7 @@ jobs:
               curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
               "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
             fi
+            "${target_python}" -m pip config set global.break-system-packages true || true
             "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
             export SETUPTOOLS_USE_DISTUTILS=local
             python_bin=$(command -v "${target_python}" || true)

From 8b31c9132f534ae7c4a0fb6ca9434ddf28612313 Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 05:43:03 +0800
Subject: [PATCH 05/14] Fix H-Coverage Python 3.12 test deps

---
 .github/workflows/H-Coverage.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/H-Coverage.yml b/.github/workflows/H-Coverage.yml
index 4622946427d67b..da7efcf5827998 100644
--- a/.github/workflows/H-Coverage.yml
+++ b/.github/workflows/H-Coverage.yml
@@ -468,7 +468,7 @@ jobs:
           }
           setup_target_python
           pip install build//python/dist/*.whl --no-deps
-          pip install -r python/unittest_py/requirements.txt
+          pip install --ignore-installed -r python/requirements.txt -r python/unittest_py/requirements.txt pytest "setuptools==68.2.0"
           bash $ci_scripts/h-test.sh
           '
 

From 249f9a5db0349d9c66a19f5180c3256bd0670834 Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 07:42:16 +0800
Subject: [PATCH 06/14] Preserve CUDA libraries for H-Coverage Fleet tests

---
 .github/workflows/H-Coverage.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/H-Coverage.yml b/.github/workflows/H-Coverage.yml
index da7efcf5827998..2a82dae9ed4b09 100644
--- a/.github/workflows/H-Coverage.yml
+++ b/.github/workflows/H-Coverage.yml
@@ -770,7 +770,7 @@ jobs:
             pip --version
           }
           setup_target_python
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
           wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
           tar -xf paddldfleet.tar.gz --strip-components=1
@@ -786,7 +786,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           set -e
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
           mkdir -p /PaddlePaddle
           cd /PaddlePaddle
           echo "Downloading Paddle.tar.gz from cfs"
@@ -805,7 +805,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -xce '
           pwd
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
           if [ "${BRANCH}" != "develop" ]; then
             git checkout $fleet_branch
             echo "Checked out fleet branch: $fleet_branch"
@@ -939,7 +939,7 @@ jobs:
             pip --version
           }
           setup_target_python
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
           wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
           tar -xf paddldfleet.tar.gz --strip-components=1
@@ -955,7 +955,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           set -e
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
           mkdir -p /PaddlePaddle
           cd /PaddlePaddle
           echo "Downloading Paddle.tar.gz from cfs"
@@ -974,7 +974,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -ce '
           export PYTHONPATH=$(pwd)
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
           if [ "${BRANCH}" != "develop" ]; then
             git checkout $fleet_branch
             echo "Checked out fleet branch: $fleet_branch"

From 7903ab4b80cec7f15129598e01ec51546fdd35ad Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 08:59:39 +0800
Subject: [PATCH 07/14] Expose NVIDIA driver libs in H-Coverage Fleet tests

---
 .github/workflows/H-Coverage.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/H-Coverage.yml b/.github/workflows/H-Coverage.yml
index 2a82dae9ed4b09..89901c73138569 100644
--- a/.github/workflows/H-Coverage.yml
+++ b/.github/workflows/H-Coverage.yml
@@ -770,7 +770,7 @@ jobs:
             pip --version
           }
           setup_target_python
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda-12.9/lib64:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
           wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
           tar -xf paddldfleet.tar.gz --strip-components=1
@@ -786,7 +786,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           set -e
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda-12.9/lib64:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
           mkdir -p /PaddlePaddle
           cd /PaddlePaddle
           echo "Downloading Paddle.tar.gz from cfs"
@@ -805,7 +805,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -xce '
           pwd
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda-12.9/lib64:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
           if [ "${BRANCH}" != "develop" ]; then
             git checkout $fleet_branch
             echo "Checked out fleet branch: $fleet_branch"
@@ -939,7 +939,7 @@ jobs:
             pip --version
           }
           setup_target_python
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda-12.9/lib64:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
           wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
           tar -xf paddldfleet.tar.gz --strip-components=1
@@ -955,7 +955,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           set -e
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda-12.9/lib64:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
           mkdir -p /PaddlePaddle
           cd /PaddlePaddle
           echo "Downloading Paddle.tar.gz from cfs"
@@ -974,7 +974,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -ce '
           export PYTHONPATH=$(pwd)
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda-12.9/lib64:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
           if [ "${BRANCH}" != "develop" ]; then
             git checkout $fleet_branch
             echo "Checked out fleet branch: $fleet_branch"

From 744cdd6a2801a6ee83a7aa8625087edaf23a0a49 Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 09:58:27 +0800
Subject: [PATCH 08/14] Use Python 3.12 Fleet image in H-Coverage

---
 .github/workflows/H-Coverage.yml | 185 ++-----------------------------
 1 file changed, 10 insertions(+), 175 deletions(-)

diff --git a/.github/workflows/H-Coverage.yml b/.github/workflows/H-Coverage.yml
index 89901c73138569..0da6d22b51e458 100644
--- a/.github/workflows/H-Coverage.yml
+++ b/.github/workflows/H-Coverage.yml
@@ -499,7 +499,7 @@ jobs:
     timeout-minutes: 60
     env:
       TASK: fleet-ci-paddle-build-whl-${{ github.event.pull_request.number }}
-      docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-dev"
+      docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
       PY_VERSION: "3.12"
     steps:
       - name: Check docker image and run container
@@ -543,63 +543,9 @@ jobs:
           rm -rf * .[^.]*
           set -x
           source ${{ github.workspace }}/../../../proxy
-          setup_target_python() {
-            local target_python="python${PY_VERSION}"
-            local target_pip="pip${PY_VERSION}"
-            local opt_python_dir="/opt/_internal/cpython-${PY_VERSION}.0/bin"
-            local apt_updated=0
-            apt_update_once() {
-              if [ "${apt_updated}" != "1" ]; then
-                apt-get update
-                apt_updated=1
-              fi
-            }
-            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
-              if [ -d "${opt_python_dir}" ]; then
-                export PATH="${opt_python_dir}:${PATH}"
-              fi
-            fi
-            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
-              apt_update_once
-              if ! apt-cache show "${target_python}" >/dev/null 2>&1; then
-                apt-get install -y software-properties-common ca-certificates curl
-                add-apt-repository -y ppa:deadsnakes/ppa
-                apt_updated=0
-                apt_update_once
-              fi
-              apt-get install -y "${target_python}" "${target_python}-dev" ca-certificates curl
-            fi
-            if ! command -v "${target_pip}" >/dev/null 2>&1; then
-              if ! command -v curl >/dev/null 2>&1; then
-                apt_update_once
-                apt-get install -y ca-certificates curl
-              fi
-              curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
-              "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
-            fi
-            "${target_python}" -m pip config set global.break-system-packages true || true
-            "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
-            export SETUPTOOLS_USE_DISTUTILS=local
-            python_bin=$(command -v "${target_python}" || true)
-            pip_bin=$(command -v "${target_pip}" || true)
-            if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
-              echo "ERROR: Python ${PY_VERSION} or pip ${PY_VERSION} is unavailable after setup." >&2
-              echo "PATH=${PATH}" >&2
-              command -v "${target_python}" || true
-              command -v "${target_pip}" || true
-              exit 1
-            fi
-            ln -sf "${python_bin}" /usr/local/bin/python
-            ln -sf "${pip_bin}" /usr/local/bin/pip
-            echo "Using python: $(command -v python)"
-            python --version
-            echo "Using pip: $(command -v pip)"
-            pip --version
-          }
-          setup_target_python
-          apt-get update
-          apt-get install -y libnvidia-compute-535
-          python -m pip install --upgrade pip wheel "setuptools==68.2.0" || python -m pip install --upgrade pip wheel "setuptools==68.2.0" --break-system-packages
+          python --version
+          pip --version
+          pip install --upgrade pip wheel
           git clone https://github.com/PaddlePaddle/PaddleFleet.git .
           git config --global --add safe.directory /paddle
           git config user.name "PaddleCI"
@@ -619,7 +565,6 @@ jobs:
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt
           echo "uv build"
           git submodule update --init --recursive
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:$LD_LIBRARY_PATH
           export IS_NVIDIA=True
           echo "Downloading Paddle.tar.gz from cfs"
           wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
@@ -680,7 +625,7 @@ jobs:
       - name: Check docker image and run container
         env:
           GPU_DEVICES: ${{ env.GPU_DEVICES }}
-          docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-dev"
+          docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
         run: |
           container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
           echo "container_name=${container_name}" >> ${{ github.env }}
@@ -716,61 +661,8 @@ jobs:
           docker exec -t ${{ env.container_name }} /bin/bash -ce '
           rm -rf * .[^.]*
           source /root/proxy
-          setup_target_python() {
-            local target_python="python${PY_VERSION}"
-            local target_pip="pip${PY_VERSION}"
-            local opt_python_dir="/opt/_internal/cpython-${PY_VERSION}.0/bin"
-            local apt_updated=0
-            apt_update_once() {
-              if [ "${apt_updated}" != "1" ]; then
-                apt-get update
-                apt_updated=1
-              fi
-            }
-            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
-              if [ -d "${opt_python_dir}" ]; then
-                export PATH="${opt_python_dir}:${PATH}"
-              fi
-            fi
-            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
-              apt_update_once
-              if ! apt-cache show "${target_python}" >/dev/null 2>&1; then
-                apt-get install -y software-properties-common ca-certificates curl
-                add-apt-repository -y ppa:deadsnakes/ppa
-                apt_updated=0
-                apt_update_once
-              fi
-              apt-get install -y "${target_python}" "${target_python}-dev" ca-certificates curl
-            fi
-            if ! command -v "${target_pip}" >/dev/null 2>&1; then
-              if ! command -v curl >/dev/null 2>&1; then
-                apt_update_once
-                apt-get install -y ca-certificates curl
-              fi
-              curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
-              "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
-            fi
-            "${target_python}" -m pip config set global.break-system-packages true || true
-            "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
-            export SETUPTOOLS_USE_DISTUTILS=local
-            python_bin=$(command -v "${target_python}" || true)
-            pip_bin=$(command -v "${target_pip}" || true)
-            if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
-              echo "ERROR: Python ${PY_VERSION} or pip ${PY_VERSION} is unavailable after setup." >&2
-              echo "PATH=${PATH}" >&2
-              command -v "${target_python}" || true
-              command -v "${target_pip}" || true
-              exit 1
-            fi
-            ln -sf "${python_bin}" /usr/local/bin/python
-            ln -sf "${pip_bin}" /usr/local/bin/pip
-            echo "Using python: $(command -v python)"
-            python --version
-            echo "Using pip: $(command -v pip)"
-            pip --version
-          }
-          setup_target_python
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda-12.9/lib64:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
+          python --version
+          pip --version
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
           wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
           tar -xf paddldfleet.tar.gz --strip-components=1
@@ -786,7 +678,6 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           set -e
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda-12.9/lib64:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
           mkdir -p /PaddlePaddle
           cd /PaddlePaddle
           echo "Downloading Paddle.tar.gz from cfs"
@@ -805,7 +696,6 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -xce '
           pwd
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda-12.9/lib64:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
           if [ "${BRANCH}" != "develop" ]; then
             git checkout $fleet_branch
             echo "Checked out fleet branch: $fleet_branch"
@@ -844,7 +734,7 @@ jobs:
     env:
       PIP_CACHE_DIR: /root/.cache/pip
       TASK: paddle-fleet-CI-${{ github.event.pull_request.number }}-multi-card_test
-      docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-dev"
+      docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
       PY_VERSION: "3.12"
     steps:
       - name: Check docker image and run container
@@ -885,61 +775,8 @@ jobs:
           docker exec -t ${{ env.container_name }} /bin/bash -ce '
           rm -rf * .[^.]*
           source /root/proxy
-          setup_target_python() {
-            local target_python="python${PY_VERSION}"
-            local target_pip="pip${PY_VERSION}"
-            local opt_python_dir="/opt/_internal/cpython-${PY_VERSION}.0/bin"
-            local apt_updated=0
-            apt_update_once() {
-              if [ "${apt_updated}" != "1" ]; then
-                apt-get update
-                apt_updated=1
-              fi
-            }
-            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
-              if [ -d "${opt_python_dir}" ]; then
-                export PATH="${opt_python_dir}:${PATH}"
-              fi
-            fi
-            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
-              apt_update_once
-              if ! apt-cache show "${target_python}" >/dev/null 2>&1; then
-                apt-get install -y software-properties-common ca-certificates curl
-                add-apt-repository -y ppa:deadsnakes/ppa
-                apt_updated=0
-                apt_update_once
-              fi
-              apt-get install -y "${target_python}" "${target_python}-dev" ca-certificates curl
-            fi
-            if ! command -v "${target_pip}" >/dev/null 2>&1; then
-              if ! command -v curl >/dev/null 2>&1; then
-                apt_update_once
-                apt-get install -y ca-certificates curl
-              fi
-              curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
-              "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
-            fi
-            "${target_python}" -m pip config set global.break-system-packages true || true
-            "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
-            export SETUPTOOLS_USE_DISTUTILS=local
-            python_bin=$(command -v "${target_python}" || true)
-            pip_bin=$(command -v "${target_pip}" || true)
-            if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
-              echo "ERROR: Python ${PY_VERSION} or pip ${PY_VERSION} is unavailable after setup." >&2
-              echo "PATH=${PATH}" >&2
-              command -v "${target_python}" || true
-              command -v "${target_pip}" || true
-              exit 1
-            fi
-            ln -sf "${python_bin}" /usr/local/bin/python
-            ln -sf "${pip_bin}" /usr/local/bin/pip
-            echo "Using python: $(command -v python)"
-            python --version
-            echo "Using pip: $(command -v pip)"
-            pip --version
-          }
-          setup_target_python
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda-12.9/lib64:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
+          python --version
+          pip --version
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
           wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
           tar -xf paddldfleet.tar.gz --strip-components=1
@@ -955,7 +792,6 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           set -e
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda-12.9/lib64:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
           mkdir -p /PaddlePaddle
           cd /PaddlePaddle
           echo "Downloading Paddle.tar.gz from cfs"
@@ -974,7 +810,6 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -ce '
           export PYTHONPATH=$(pwd)
-          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda-12.9/lib64:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
           if [ "${BRANCH}" != "develop" ]; then
             git checkout $fleet_branch
             echo "Checked out fleet branch: $fleet_branch"

From ce5bf4d4850e175e8fcea3e0dd6e66c1185a2622 Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 10:16:07 +0800
Subject: [PATCH 09/14] Remove Python setup hacks from H-Coverage

---
 .github/workflows/H-Coverage.yml | 118 ++-----------------------------
 1 file changed, 7 insertions(+), 111 deletions(-)

diff --git a/.github/workflows/H-Coverage.yml b/.github/workflows/H-Coverage.yml
index 0da6d22b51e458..8735ab7f48a9f0 100644
--- a/.github/workflows/H-Coverage.yml
+++ b/.github/workflows/H-Coverage.yml
@@ -113,7 +113,7 @@ jobs:
         run: |
           container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S)
           echo "container_name=${container_name}" >> ${{ github.env }}
-          docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda129-coverage-test
+          docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev
           docker run -d -t --name ${container_name} \
             -v "/home/data/cfs:/home/data/cfs" \
             -v "/home/data/cfs/.cache:/root/.cache" \
@@ -192,60 +192,8 @@ jobs:
           mkdir -p ${CFS_DIR}/.cache/coverage
           mkdir -p ${CFS_DIR}/.ccache/coverage
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-12.9/compat
-          setup_target_python() {
-            local target_python="python${PY_VERSION}"
-            local target_pip="pip${PY_VERSION}"
-            local opt_python_dir="/opt/_internal/cpython-${PY_VERSION}.0/bin"
-            local apt_updated=0
-            apt_update_once() {
-              if [ "${apt_updated}" != "1" ]; then
-                apt-get update
-                apt_updated=1
-              fi
-            }
-            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
-              if [ -d "${opt_python_dir}" ]; then
-                export PATH="${opt_python_dir}:${PATH}"
-              fi
-            fi
-            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
-              apt_update_once
-              if ! apt-cache show "${target_python}" >/dev/null 2>&1; then
-                apt-get install -y software-properties-common ca-certificates curl
-                add-apt-repository -y ppa:deadsnakes/ppa
-                apt_updated=0
-                apt_update_once
-              fi
-              apt-get install -y "${target_python}" "${target_python}-dev" ca-certificates curl
-            fi
-            if ! command -v "${target_pip}" >/dev/null 2>&1; then
-              if ! command -v curl >/dev/null 2>&1; then
-                apt_update_once
-                apt-get install -y ca-certificates curl
-              fi
-              curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
-              "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
-            fi
-            "${target_python}" -m pip config set global.break-system-packages true || true
-            "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
-            export SETUPTOOLS_USE_DISTUTILS=local
-            python_bin=$(command -v "${target_python}" || true)
-            pip_bin=$(command -v "${target_pip}" || true)
-            if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
-              echo "ERROR: Python ${PY_VERSION} or pip ${PY_VERSION} is unavailable after setup." >&2
-              echo "PATH=${PATH}" >&2
-              command -v "${target_python}" || true
-              command -v "${target_pip}" || true
-              exit 1
-            fi
-            ln -sf "${python_bin}" /usr/local/bin/python
-            ln -sf "${pip_bin}" /usr/local/bin/pip
-            echo "Using python: $(command -v python)"
-            python --version
-            echo "Using pip: $(command -v pip)"
-            pip --version
-          }
-          setup_target_python
+          python --version
+          pip --version
           bash ${ci_scripts}/cmake-predownload.sh
           bash ${ci_scripts}/cmake_preload_data_file.sh
           pip install -r python/requirements.txt
@@ -352,7 +300,7 @@ jobs:
         run: |
           container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
           echo "container_name=${container_name}" >> ${{ github.env }}
-          docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda129-coverage-test
+          docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev
           docker run -d -t --gpus "\"device=${GPU_DEVICES}\"" --name ${container_name} \
             -v "/home/data/cfs:/home/data/cfs" \
             -v "/home/data/cfs/.cache:/root/.cache" \
@@ -413,62 +361,10 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           source ${{ github.workspace }}/../../../proxy
-          setup_target_python() {
-            local target_python="python${PY_VERSION}"
-            local target_pip="pip${PY_VERSION}"
-            local opt_python_dir="/opt/_internal/cpython-${PY_VERSION}.0/bin"
-            local apt_updated=0
-            apt_update_once() {
-              if [ "${apt_updated}" != "1" ]; then
-                apt-get update
-                apt_updated=1
-              fi
-            }
-            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
-              if [ -d "${opt_python_dir}" ]; then
-                export PATH="${opt_python_dir}:${PATH}"
-              fi
-            fi
-            if ! command -v "${target_python}" >/dev/null 2>&1 || ! command -v "${target_pip}" >/dev/null 2>&1; then
-              apt_update_once
-              if ! apt-cache show "${target_python}" >/dev/null 2>&1; then
-                apt-get install -y software-properties-common ca-certificates curl
-                add-apt-repository -y ppa:deadsnakes/ppa
-                apt_updated=0
-                apt_update_once
-              fi
-              apt-get install -y "${target_python}" "${target_python}-dev" ca-certificates curl
-            fi
-            if ! command -v "${target_pip}" >/dev/null 2>&1; then
-              if ! command -v curl >/dev/null 2>&1; then
-                apt_update_once
-                apt-get install -y ca-certificates curl
-              fi
-              curl -fsSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
-              "${target_python}" /tmp/get-pip.py || "${target_python}" /tmp/get-pip.py --break-system-packages
-            fi
-            "${target_python}" -m pip config set global.break-system-packages true || true
-            "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel || "${target_python}" -m pip install --upgrade "setuptools==68.2.0" wheel --break-system-packages
-            export SETUPTOOLS_USE_DISTUTILS=local
-            python_bin=$(command -v "${target_python}" || true)
-            pip_bin=$(command -v "${target_pip}" || true)
-            if [ -z "${python_bin}" ] || [ -z "${pip_bin}" ]; then
-              echo "ERROR: Python ${PY_VERSION} or pip ${PY_VERSION} is unavailable after setup." >&2
-              echo "PATH=${PATH}" >&2
-              command -v "${target_python}" || true
-              command -v "${target_pip}" || true
-              exit 1
-            fi
-            ln -sf "${python_bin}" /usr/local/bin/python
-            ln -sf "${pip_bin}" /usr/local/bin/pip
-            echo "Using python: $(command -v python)"
-            python --version
-            echo "Using pip: $(command -v pip)"
-            pip --version
-          }
-          setup_target_python
+          python --version
+          pip --version
           pip install build//python/dist/*.whl --no-deps
-          pip install --ignore-installed -r python/requirements.txt -r python/unittest_py/requirements.txt pytest "setuptools==68.2.0"
+          pip install -r python/unittest_py/requirements.txt
           bash $ci_scripts/h-test.sh
           '
 

From a88d57aff43edb37deba32ece7d1e0c967343907 Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 13:22:59 +0800
Subject: [PATCH 10/14] fix(ci): install pytest for H-Coverage FA tests

---
 .github/workflows/H-Coverage.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/H-Coverage.yml b/.github/workflows/H-Coverage.yml
index 8735ab7f48a9f0..9c77402545e5d8 100644
--- a/.github/workflows/H-Coverage.yml
+++ b/.github/workflows/H-Coverage.yml
@@ -373,6 +373,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           source ${{ github.workspace }}/../../../proxy
+          python -m pip install pytest
           cd test/test_flashmask_ci
           bash run.sh
           '

From 0ad83ab8ca8132ce17c8903d039ddfcde1898a48 Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 13:38:35 +0800
Subject: [PATCH 11/14] fix(ci): restore minimal H-Coverage setup

---
 .github/workflows/H-Coverage.yml | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/H-Coverage.yml b/.github/workflows/H-Coverage.yml
index 9c77402545e5d8..d43d8126491221 100644
--- a/.github/workflows/H-Coverage.yml
+++ b/.github/workflows/H-Coverage.yml
@@ -179,7 +179,6 @@ jobs:
       - name: Build
         run: |
           docker exec -t ${{ env.container_name }}  /bin/bash -c '
-          source ${{ github.workspace }}/../../../proxy
           flashattn_version=$(git submodule status|grep flashattn|awk "{print \$1}"|sed "s#-##g")
           echo flashattn_version:$flashattn_version
           url="https://xly-devops.bj.bcebos.com/gpups/flash-attention/cu90/flashattn_libs_${flashattn_version}.tar"
@@ -192,8 +191,7 @@ jobs:
           mkdir -p ${CFS_DIR}/.cache/coverage
           mkdir -p ${CFS_DIR}/.ccache/coverage
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-12.9/compat
-          python --version
-          pip --version
+          source ${{ github.workspace }}/../../../proxy
           bash ${ci_scripts}/cmake-predownload.sh
           bash ${ci_scripts}/cmake_preload_data_file.sh
           pip install -r python/requirements.txt
@@ -361,8 +359,6 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           source ${{ github.workspace }}/../../../proxy
-          python --version
-          pip --version
           pip install build//python/dist/*.whl --no-deps
           pip install -r python/unittest_py/requirements.txt
           bash $ci_scripts/h-test.sh
@@ -440,9 +436,9 @@ jobs:
           rm -rf * .[^.]*
           set -x
           source ${{ github.workspace }}/../../../proxy
-          python --version
-          pip --version
-          pip install --upgrade pip wheel
+          apt-get update
+          apt-get install -y libnvidia-compute-535
+          python -m pip install --upgrade pip wheel setuptools
           git clone https://github.com/PaddlePaddle/PaddleFleet.git .
           git config --global --add safe.directory /paddle
           git config user.name "PaddleCI"
@@ -462,6 +458,7 @@ jobs:
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt
           echo "uv build"
           git submodule update --init --recursive
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:$LD_LIBRARY_PATH
           export IS_NVIDIA=True
           echo "Downloading Paddle.tar.gz from cfs"
           wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
@@ -558,8 +555,7 @@ jobs:
           docker exec -t ${{ env.container_name }} /bin/bash -ce '
           rm -rf * .[^.]*
           source /root/proxy
-          python --version
-          pip --version
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
           wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
           tar -xf paddldfleet.tar.gz --strip-components=1
@@ -575,6 +571,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           set -e
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
           mkdir -p /PaddlePaddle
           cd /PaddlePaddle
           echo "Downloading Paddle.tar.gz from cfs"
@@ -593,6 +590,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -xce '
           pwd
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
           if [ "${BRANCH}" != "develop" ]; then
             git checkout $fleet_branch
             echo "Checked out fleet branch: $fleet_branch"
@@ -672,8 +670,7 @@ jobs:
           docker exec -t ${{ env.container_name }} /bin/bash -ce '
           rm -rf * .[^.]*
           source /root/proxy
-          python --version
-          pip --version
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
           pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
           wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
           tar -xf paddldfleet.tar.gz --strip-components=1
@@ -689,6 +686,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           set -e
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
           mkdir -p /PaddlePaddle
           cd /PaddlePaddle
           echo "Downloading Paddle.tar.gz from cfs"
@@ -707,6 +705,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -ce '
           export PYTHONPATH=$(pwd)
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
           if [ "${BRANCH}" != "develop" ]; then
             git checkout $fleet_branch
             echo "Checked out fleet branch: $fleet_branch"

From d4dccd931243d250eb36eed40723b672b883c613 Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 16:21:29 +0800
Subject: [PATCH 12/14] fix(ci): make H-Coverage FA tests fail reliably

---
 .github/workflows/H-Coverage.yml         |  2 +-
 test/test_flashmask_ci/flashmask_gt.json | 92 ++++++++++++------------
 test/test_flashmask_ci/run.sh            |  2 +
 3 files changed, 49 insertions(+), 47 deletions(-)

diff --git a/.github/workflows/H-Coverage.yml b/.github/workflows/H-Coverage.yml
index d43d8126491221..766fd8092fb9fa 100644
--- a/.github/workflows/H-Coverage.yml
+++ b/.github/workflows/H-Coverage.yml
@@ -369,7 +369,7 @@ jobs:
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           source ${{ github.workspace }}/../../../proxy
-          python -m pip install pytest
+          python -m pip install pytest einops
           cd test/test_flashmask_ci
           bash run.sh
           '
diff --git a/test/test_flashmask_ci/flashmask_gt.json b/test/test_flashmask_ci/flashmask_gt.json
index e644d6e4e2faff..768a3c2f1df60c 100644
--- a/test/test_flashmask_ci/flashmask_gt.json
+++ b/test/test_flashmask_ci/flashmask_gt.json
@@ -19,16 +19,16 @@
   "gen_startend_row_indices7-1-8192-33792-2-1-1-80-80-3-dtype0": "6f8a11a2c3cbf189a274d9879e4f7e74",
   "gen_startend_row_indices8-1-8192-33792-2-1-1-80-80-3-dtype0": "223aec02c8e609f149b66eee8ef85f16",
   "gen_startend_row_indices9-1-8192-33792-2-1-1-80-80-3-dtype0": "276e98f28141a669d59e4656e02b3c7a",
-  "gen_startend_row_indices0-1-8192-33792-2-1-1-64-64-3-dtype0": "832fe895d299ba62bb3cf2a1c585090b",
-  "gen_startend_row_indices1-1-8192-33792-2-1-1-64-64-3-dtype0": "4dd468c9e48c0c9fb2444856c9d01b44",
-  "gen_startend_row_indices2-1-8192-33792-2-1-1-64-64-3-dtype0": "c7fbf1816774c1553fc506fa805baced",
-  "gen_startend_row_indices3-1-8192-33792-2-1-1-64-64-3-dtype0": "4dd468c9e48c0c9fb2444856c9d01b44",
-  "gen_startend_row_indices4-1-8192-33792-2-1-1-64-64-3-dtype0": "c67dcd5e0d4523268e1cb75220826fa5",
-  "gen_startend_row_indices5-1-8192-33792-2-1-1-64-64-3-dtype0": "4dd468c9e48c0c9fb2444856c9d01b44",
-  "gen_startend_row_indices6-1-8192-33792-2-1-1-64-64-3-dtype0": "e147c70967201739ae835f07c9d65bdd",
-  "gen_startend_row_indices7-1-8192-33792-2-1-1-64-64-3-dtype0": "e147c70967201739ae835f07c9d65bdd",
-  "gen_startend_row_indices8-1-8192-33792-2-1-1-64-64-3-dtype0": "e3885302eb4fbf8f930c018d1ab024f4",
-  "gen_startend_row_indices9-1-8192-33792-2-1-1-64-64-3-dtype0": "4dd468c9e48c0c9fb2444856c9d01b44",
+  "gen_startend_row_indices0-1-8192-33792-2-1-1-64-64-3-dtype0": "de0e0438bf1929e4dc145a9cae6c65cb",
+  "gen_startend_row_indices1-1-8192-33792-2-1-1-64-64-3-dtype0": "6354d399ab9a4a13dd6e35be6ca608cc",
+  "gen_startend_row_indices2-1-8192-33792-2-1-1-64-64-3-dtype0": "8d54745393a55eb2eb697c309b9c7985",
+  "gen_startend_row_indices3-1-8192-33792-2-1-1-64-64-3-dtype0": "6354d399ab9a4a13dd6e35be6ca608cc",
+  "gen_startend_row_indices4-1-8192-33792-2-1-1-64-64-3-dtype0": "136464684753a961677d6ac95cb12363",
+  "gen_startend_row_indices5-1-8192-33792-2-1-1-64-64-3-dtype0": "6354d399ab9a4a13dd6e35be6ca608cc",
+  "gen_startend_row_indices6-1-8192-33792-2-1-1-64-64-3-dtype0": "3c758670d54ee7729af549a6bfbef101",
+  "gen_startend_row_indices7-1-8192-33792-2-1-1-64-64-3-dtype0": "3c758670d54ee7729af549a6bfbef101",
+  "gen_startend_row_indices8-1-8192-33792-2-1-1-64-64-3-dtype0": "3ac600b75cf803abd03cc2124082ec62",
+  "gen_startend_row_indices9-1-8192-33792-2-1-1-64-64-3-dtype0": "6354d399ab9a4a13dd6e35be6ca608cc",
   "gen_startend_row_indices0-2840-32-32-16-4-1-128-128-3-dtype0": "75d4604f539bdeef39496544e3955008",
   "gen_startend_row_indices1-2840-32-32-16-4-1-128-128-3-dtype0": "1daf514cc698acb8449338629b4991e2",
   "gen_startend_row_indices2-2840-32-32-16-4-1-128-128-3-dtype0": "8624e86be21843c69bebde6e59c616ee",
@@ -109,16 +109,16 @@
   "gen_startend_row_indices7-1-300-300-16-16-1-80-80-3-dtype0": "1c2dfc027a7d1ff8a157f3158417f4a6",
   "gen_startend_row_indices8-1-300-300-16-16-1-80-80-3-dtype0": "83e8b2752bee81fac3ec6d51761c1459",
   "gen_startend_row_indices9-1-300-300-16-16-1-80-80-3-dtype0": "3d6991c67167de52846d75bafd637eb6",
-  "gen_startend_row_indices0-1-300-300-16-16-1-64-64-3-dtype0": "c3d5431321250abeaddc1237172b9e5c",
-  "gen_startend_row_indices1-1-300-300-16-16-1-64-64-3-dtype0": "6ff85ceae3f4b74b73fcec2d4b05eaa7",
-  "gen_startend_row_indices2-1-300-300-16-16-1-64-64-3-dtype0": "c16b31de4b90f5c01269628ceca82235",
-  "gen_startend_row_indices3-1-300-300-16-16-1-64-64-3-dtype0": "f74686d25f3e034b4166853324a3b237",
-  "gen_startend_row_indices4-1-300-300-16-16-1-64-64-3-dtype0": "c4834536e28f3fda10fc38a269023e76",
-  "gen_startend_row_indices5-1-300-300-16-16-1-64-64-3-dtype0": "7b7fa4996eb2a037180601de80da8fb2",
-  "gen_startend_row_indices6-1-300-300-16-16-1-64-64-3-dtype0": "6f3f0cca7f0242e49b9adb8fab16a55c",
-  "gen_startend_row_indices7-1-300-300-16-16-1-64-64-3-dtype0": "ee85640576bbc7089a697ef68d452263",
-  "gen_startend_row_indices8-1-300-300-16-16-1-64-64-3-dtype0": "e7fd028a89db9e925d9fe06a37ae1749",
-  "gen_startend_row_indices9-1-300-300-16-16-1-64-64-3-dtype0": "9cb6fd1607399c3e4767616021ad1955",
+  "gen_startend_row_indices0-1-300-300-16-16-1-64-64-3-dtype0": "3c4a8e5526e4020791e09da80b9b443f",
+  "gen_startend_row_indices1-1-300-300-16-16-1-64-64-3-dtype0": "e3bc435c882efb28862836dd4a84912b",
+  "gen_startend_row_indices2-1-300-300-16-16-1-64-64-3-dtype0": "52dac3b8053b40d2f73c5df954c192b3",
+  "gen_startend_row_indices3-1-300-300-16-16-1-64-64-3-dtype0": "da310e73d0de2062f7a0ed870b663548",
+  "gen_startend_row_indices4-1-300-300-16-16-1-64-64-3-dtype0": "63354fb212f95a81fa8e87689d4d87e1",
+  "gen_startend_row_indices5-1-300-300-16-16-1-64-64-3-dtype0": "2d6d451c261b2f802824cb7594d1b9f8",
+  "gen_startend_row_indices6-1-300-300-16-16-1-64-64-3-dtype0": "414e667be64dc78448d1284f04ceeec3",
+  "gen_startend_row_indices7-1-300-300-16-16-1-64-64-3-dtype0": "90aceddc48e9273e27694d2f6162e659",
+  "gen_startend_row_indices8-1-300-300-16-16-1-64-64-3-dtype0": "0b4ee542fa7732a235e1efc350bba901",
+  "gen_startend_row_indices9-1-300-300-16-16-1-64-64-3-dtype0": "ac21da119782ac451fbb7e0ae6789cde",
   "gen_startend_row_indices0-1-300-300-16-16-16-128-128-3-dtype0": "e829012daa321ea9f1956bc264e4a4f6",
   "gen_startend_row_indices1-1-300-300-16-16-16-128-128-3-dtype0": "60338d7811fe9bf2682ce40b72af631b",
   "gen_startend_row_indices2-1-300-300-16-16-16-128-128-3-dtype0": "225f3424253cc5d61c20aaa72e3717ae",
@@ -139,16 +139,16 @@
   "gen_startend_row_indices7-1-300-300-16-16-16-80-80-3-dtype0": "1c2dfc027a7d1ff8a157f3158417f4a6",
   "gen_startend_row_indices8-1-300-300-16-16-16-80-80-3-dtype0": "83e8b2752bee81fac3ec6d51761c1459",
   "gen_startend_row_indices9-1-300-300-16-16-16-80-80-3-dtype0": "998daf470bd932da27db90f4bccba87f",
-  "gen_startend_row_indices0-1-300-300-16-16-16-64-64-3-dtype0": "c3d5431321250abeaddc1237172b9e5c",
-  "gen_startend_row_indices1-1-300-300-16-16-16-64-64-3-dtype0": "6ff85ceae3f4b74b73fcec2d4b05eaa7",
-  "gen_startend_row_indices2-1-300-300-16-16-16-64-64-3-dtype0": "c16b31de4b90f5c01269628ceca82235",
-  "gen_startend_row_indices3-1-300-300-16-16-16-64-64-3-dtype0": "f74686d25f3e034b4166853324a3b237",
-  "gen_startend_row_indices4-1-300-300-16-16-16-64-64-3-dtype0": "c4834536e28f3fda10fc38a269023e76",
-  "gen_startend_row_indices5-1-300-300-16-16-16-64-64-3-dtype0": "7b7fa4996eb2a037180601de80da8fb2",
-  "gen_startend_row_indices6-1-300-300-16-16-16-64-64-3-dtype0": "6f3f0cca7f0242e49b9adb8fab16a55c",
-  "gen_startend_row_indices7-1-300-300-16-16-16-64-64-3-dtype0": "ee85640576bbc7089a697ef68d452263",
-  "gen_startend_row_indices8-1-300-300-16-16-16-64-64-3-dtype0": "e7fd028a89db9e925d9fe06a37ae1749",
-  "gen_startend_row_indices9-1-300-300-16-16-16-64-64-3-dtype0": "709e70e253e6e0e6204f63fde72e0d5b",
+  "gen_startend_row_indices0-1-300-300-16-16-16-64-64-3-dtype0": "3c4a8e5526e4020791e09da80b9b443f",
+  "gen_startend_row_indices1-1-300-300-16-16-16-64-64-3-dtype0": "e3bc435c882efb28862836dd4a84912b",
+  "gen_startend_row_indices2-1-300-300-16-16-16-64-64-3-dtype0": "52dac3b8053b40d2f73c5df954c192b3",
+  "gen_startend_row_indices3-1-300-300-16-16-16-64-64-3-dtype0": "da310e73d0de2062f7a0ed870b663548",
+  "gen_startend_row_indices4-1-300-300-16-16-16-64-64-3-dtype0": "63354fb212f95a81fa8e87689d4d87e1",
+  "gen_startend_row_indices5-1-300-300-16-16-16-64-64-3-dtype0": "2d6d451c261b2f802824cb7594d1b9f8",
+  "gen_startend_row_indices6-1-300-300-16-16-16-64-64-3-dtype0": "414e667be64dc78448d1284f04ceeec3",
+  "gen_startend_row_indices7-1-300-300-16-16-16-64-64-3-dtype0": "90aceddc48e9273e27694d2f6162e659",
+  "gen_startend_row_indices8-1-300-300-16-16-16-64-64-3-dtype0": "0b4ee542fa7732a235e1efc350bba901",
+  "gen_startend_row_indices9-1-300-300-16-16-16-64-64-3-dtype0": "a3ea8a49219141eff99dd532b7907f4a",
   "gen_startend_row_indices0-1-128-127-1-1-1-128-128-3-dtype0": "da0eee3fdf461e6f015b5360595c30ef",
   "gen_startend_row_indices1-1-128-127-1-1-1-128-128-3-dtype0": "ec86715f0f0ba520fab44cf4e05e0f60",
   "gen_startend_row_indices2-1-128-127-1-1-1-128-128-3-dtype0": "ea4407cbfad4b8d86c2031430011d3f4",
@@ -169,16 +169,16 @@
   "gen_startend_row_indices7-1-128-127-1-1-1-80-80-3-dtype0": "30e47c3513d68703fe233a8f96006e96",
   "gen_startend_row_indices8-1-128-127-1-1-1-80-80-3-dtype0": "f8f8efef3816d5686c84297003e08677",
   "gen_startend_row_indices9-1-128-127-1-1-1-80-80-3-dtype0": "b318eb75c32d18dc452f4c5eaa303c9d",
-  "gen_startend_row_indices0-1-128-127-1-1-1-64-64-3-dtype0": "1ae7d925db2f3d4728c347ff5a98edd8",
-  "gen_startend_row_indices1-1-128-127-1-1-1-64-64-3-dtype0": "1cb070d2b56c09d1c2a7473616ca8fe4",
+  "gen_startend_row_indices0-1-128-127-1-1-1-64-64-3-dtype0": "4031fbc9e9d39f6ad46ea19baf306522",
+  "gen_startend_row_indices1-1-128-127-1-1-1-64-64-3-dtype0": "f1504910842d96eb11a9ab2c8281660a",
   "gen_startend_row_indices2-1-128-127-1-1-1-64-64-3-dtype0": "c59f6e18d0e3d8f55794cbcd4ca086fe",
-  "gen_startend_row_indices3-1-128-127-1-1-1-64-64-3-dtype0": "d14ea041ff8ccec0a09c25baaa081803",
+  "gen_startend_row_indices3-1-128-127-1-1-1-64-64-3-dtype0": "6d378a97f83a37dcd69d0ac4449c6988",
   "gen_startend_row_indices4-1-128-127-1-1-1-64-64-3-dtype0": "78e535b25cef325d6f47cb7b9ee112e1",
-  "gen_startend_row_indices5-1-128-127-1-1-1-64-64-3-dtype0": "85818416e2c045a03f404b652e8d45d0",
+  "gen_startend_row_indices5-1-128-127-1-1-1-64-64-3-dtype0": "c5dd3a733b3662ac5ce34fd8329babb0",
   "gen_startend_row_indices6-1-128-127-1-1-1-64-64-3-dtype0": "927e978de57a5595471f98bc557949e7",
   "gen_startend_row_indices7-1-128-127-1-1-1-64-64-3-dtype0": "00cb19241518fb50f21ed3c609026290",
-  "gen_startend_row_indices8-1-128-127-1-1-1-64-64-3-dtype0": "401b48780b94781e4712fbda88786c99",
-  "gen_startend_row_indices9-1-128-127-1-1-1-64-64-3-dtype0": "6cd6fdf42043bab1c4a0b328502550ab",
+  "gen_startend_row_indices8-1-128-127-1-1-1-64-64-3-dtype0": "2676428b36aca250f277d39ddbd51fd0",
+  "gen_startend_row_indices9-1-128-127-1-1-1-64-64-3-dtype0": "8555f39d0c338602ae527049f3bb9c80",
   "gen_startend_row_indices0-2-16384-16383-4-1-1-128-128-3-dtype0": "028adc4ce9cd9b28ee5f33e828f4c8f9",
   "gen_startend_row_indices1-2-16384-16383-4-1-1-128-128-3-dtype0": "d7b370be62693319b6435f22a0ba3590",
   "gen_startend_row_indices2-2-16384-16383-4-1-1-128-128-3-dtype0": "463d09dd28b2ebdda6dd1fcf0f0bdd28",
@@ -199,16 +199,16 @@
   "gen_startend_row_indices7-2-16384-16383-4-1-1-80-80-3-dtype0": "8ff4f1415529010c719a87766babf63a",
   "gen_startend_row_indices8-2-16384-16383-4-1-1-80-80-3-dtype0": "e7ef54354c7a4d8d2dd66dbb822d1897",
   "gen_startend_row_indices9-2-16384-16383-4-1-1-80-80-3-dtype0": "84972da6ab2f99c01bb637ffeb621696",
-  "gen_startend_row_indices0-2-16384-16383-4-1-1-64-64-3-dtype0": "d3865daa03276277b52bcf837c940472",
-  "gen_startend_row_indices1-2-16384-16383-4-1-1-64-64-3-dtype0": "1a3cc3a4c03aa69e40676a47d3966f81",
-  "gen_startend_row_indices2-2-16384-16383-4-1-1-64-64-3-dtype0": "54e656dedb77473f30c91b7fc5efb622",
-  "gen_startend_row_indices3-2-16384-16383-4-1-1-64-64-3-dtype0": "97f4aeed9f570b189b7e2e95b54d094b",
-  "gen_startend_row_indices4-2-16384-16383-4-1-1-64-64-3-dtype0": "06a623fba8de963ed78a5dbd667cba52",
-  "gen_startend_row_indices5-2-16384-16383-4-1-1-64-64-3-dtype0": "ea4e4de2f6aaa076998ac1cd89d7cbf3",
-  "gen_startend_row_indices6-2-16384-16383-4-1-1-64-64-3-dtype0": "e044ff6bb7c06867f1d23de52f72018b",
-  "gen_startend_row_indices7-2-16384-16383-4-1-1-64-64-3-dtype0": "a51955812865e8e98356f135d1bd3abd",
-  "gen_startend_row_indices8-2-16384-16383-4-1-1-64-64-3-dtype0": "562b6f426a76df618a1e845b0f2ab85f",
-  "gen_startend_row_indices9-2-16384-16383-4-1-1-64-64-3-dtype0": "989deffab5b0ccf074f90adc8a8fecbd",
+  "gen_startend_row_indices0-2-16384-16383-4-1-1-64-64-3-dtype0": "b1143dff78a476ed923e58c5d6844e81",
+  "gen_startend_row_indices1-2-16384-16383-4-1-1-64-64-3-dtype0": "f825f1299dba84f2ce4330de3b5e29d4",
+  "gen_startend_row_indices2-2-16384-16383-4-1-1-64-64-3-dtype0": "00527259e57a4ef8cecbad406be732e0",
+  "gen_startend_row_indices3-2-16384-16383-4-1-1-64-64-3-dtype0": "25746353628ff2264a06456fbcbfe235",
+  "gen_startend_row_indices4-2-16384-16383-4-1-1-64-64-3-dtype0": "860dec17ecc61e30d30a38cebdcef726",
+  "gen_startend_row_indices5-2-16384-16383-4-1-1-64-64-3-dtype0": "10291567208228555ef82b269d3a8623",
+  "gen_startend_row_indices6-2-16384-16383-4-1-1-64-64-3-dtype0": "5ca25f3763cda372cc5856ac89bea719",
+  "gen_startend_row_indices7-2-16384-16383-4-1-1-64-64-3-dtype0": "77210e4dbc776a272bd7ce9a235891cd",
+  "gen_startend_row_indices8-2-16384-16383-4-1-1-64-64-3-dtype0": "47d3c84b4ff075b053b7e8075185685e",
+  "gen_startend_row_indices9-2-16384-16383-4-1-1-64-64-3-dtype0": "ca64e369235b9a1a001c7e174421ffe9",
   "gt_commit_id": "61459524eb45ac1fd8ec1d75be38ad47c410c756",
   "gt_commit_msg": "add blockmask for FlashMaskV3 (#89)"
 }
diff --git a/test/test_flashmask_ci/run.sh b/test/test_flashmask_ci/run.sh
index b6bee0291b7b10..8a0f14eb1e353f 100644
--- a/test/test_flashmask_ci/run.sh
+++ b/test/test_flashmask_ci/run.sh
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -e
+
 python -m pytest test_flashmask_ci.py -v
 python -m pytest test_fwd_md5sum.py -v
 python -m pytest test_flashmask_group.py -v

From 420d682c1761e37cd61c8a6e998236f90740ef74 Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 18:12:14 +0800
Subject: [PATCH 13/14] test: skip unsupported FlashMask V3 sparse backward

---
 test/test_flashmask_ci/test_flashmask_ci.py | 23 +++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/test/test_flashmask_ci/test_flashmask_ci.py b/test/test_flashmask_ci/test_flashmask_ci.py
index 5704c330837ac1..96e29fb513bbee 100644
--- a/test/test_flashmask_ci/test_flashmask_ci.py
+++ b/test/test_flashmask_ci/test_flashmask_ci.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from functools import partial
+from functools import cache, partial
 
 import pytest
 from generate_startend_row_indices import (
@@ -32,7 +32,16 @@
 from test_util import attention_ref
 
 import paddle
-from paddle.nn.functional.flash_attention import flashmask_attention
+from paddle.nn.functional.flash_attention import (
+    flashmask_attention,
+    flashmask_get_unique_id,
+)
+
+
+@cache
+def _has_flashmask_v3_sparse_backward_support():
+    return bool((flashmask_get_unique_id().numpy() != 0).any())
+
 
 # batch_size, seqlen_q, seqlen_k, nheads, nheads_kv
 shape_cases = [
@@ -143,6 +152,16 @@ def test_flashmask(
         batch_size, seqlen_q, seqlen_k, nheads_startend_row_indices
     )
 
+    if (
+        fa_version == 3
+        and startend_row_indices is not None
+        and not _has_flashmask_v3_sparse_backward_support()
+    ):
+        pytest.skip(
+            "Skipping FlashMask V3 sparse-mask backward because "
+            "NVSHMEM/FlashMask V3 sparse backward support is unavailable."
+        )
+
     if startend_row_indices is None and causal and d == 80:
         pytest.skip(
             "Skipping because running headdim 80 with flash_attn in causal mask"

From 175d43b72937821819d4467b64e02057c5fd0ff4 Mon Sep 17 00:00:00 2001
From: Nyako Shigure <shigure_nyako@outlook.com>
Date: Mon, 1 Jun 2026 20:05:27 +0800
Subject: [PATCH 14/14] test: revert FlashMask V3 sparse backward skip

---
 test/test_flashmask_ci/test_flashmask_ci.py | 23 ++-------------------
 1 file changed, 2 insertions(+), 21 deletions(-)

diff --git a/test/test_flashmask_ci/test_flashmask_ci.py b/test/test_flashmask_ci/test_flashmask_ci.py
index 96e29fb513bbee..5704c330837ac1 100644
--- a/test/test_flashmask_ci/test_flashmask_ci.py
+++ b/test/test_flashmask_ci/test_flashmask_ci.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from functools import cache, partial
+from functools import partial
 
 import pytest
 from generate_startend_row_indices import (
@@ -32,16 +32,7 @@
 from test_util import attention_ref
 
 import paddle
-from paddle.nn.functional.flash_attention import (
-    flashmask_attention,
-    flashmask_get_unique_id,
-)
-
-
-@cache
-def _has_flashmask_v3_sparse_backward_support():
-    return bool((flashmask_get_unique_id().numpy() != 0).any())
-
+from paddle.nn.functional.flash_attention import flashmask_attention
 
 # batch_size, seqlen_q, seqlen_k, nheads, nheads_kv
 shape_cases = [
@@ -152,16 +143,6 @@ def test_flashmask(
         batch_size, seqlen_q, seqlen_k, nheads_startend_row_indices
     )
 
-    if (
-        fa_version == 3
-        and startend_row_indices is not None
-        and not _has_flashmask_v3_sparse_backward_support()
-    ):
-        pytest.skip(
-            "Skipping FlashMask V3 sparse-mask backward because "
-            "NVSHMEM/FlashMask V3 sparse backward support is unavailable."
-        )
-
     if startend_row_indices is None and causal and d == 80:
         pytest.skip(
             "Skipping because running headdim 80 with flash_attn in causal mask"