Preserve configured gradient clipping in accuracy mode

.dev_scripts/ci_source_copy.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+if [ "$#" -lt 2 ]; then
+  echo "Usage: $0 SOURCE_DIR COMMAND [ARG...]" >&2
+  exit 2
+fi
+
+source_dir=$1
+shift
+test_dir="$(mktemp -d /tmp/swift-ci.XXXXXX)"
+trap 'rm -rf -- "${test_dir:?}"' EXIT
+
+# Build products and Git changes stay in the container, owned by its test user.
+cp -R "$source_dir/." "$test_dir/"
+cd "$test_dir"
+"$@"

.dev_scripts/dockerci.sh

@@ -2,6 +2,7 @@
 MODELSCOPE_CACHE_DIR_IN_CONTAINER=/modelscope_cache
 CODE_DIR=$PWD
 CODE_DIR_IN_CONTAINER=/ms-swift
+SOURCE_DIR_IN_CONTAINER=/ms-swift-source
 MODELSCOPE_SDK_DEBUG=True
 echo "$USER"
 gpus='0,1 2,3'
@@ -31,7 +32,7 @@ do
     docker run --rm --name $CONTAINER_NAME --shm-size=16gb \
               --cpuset-cpus=${cpu_sets_arr[$idx]} \
               --gpus='"'"device=$gpu"'"' \
-              -v $CODE_DIR:$CODE_DIR_IN_CONTAINER \
+              -v "$CODE_DIR:$SOURCE_DIR_IN_CONTAINER:ro" \
               -v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
               -v $MODELSCOPE_HOME_CACHE/$idx:/root \
               -v /home/admin/pre-commit:/home/admin/pre-commit \
@@ -51,12 +52,13 @@ do
 	            -e PR_CHANGED_FILES=$PR_CHANGED_FILES \
               --workdir=$CODE_DIR_IN_CONTAINER \
               ${IMAGE_NAME}:${IMAGE_VERSION} \
-              $CI_COMMAND
+              bash "$SOURCE_DIR_IN_CONTAINER/.dev_scripts/ci_source_copy.sh" \
+              "$SOURCE_DIR_IN_CONTAINER" $CI_COMMAND
   else
     docker run --rm --name $CONTAINER_NAME --shm-size=16gb \
               --cpuset-cpus=${cpu_sets_arr[$idx]} \
               --gpus='"'"device=$gpu"'"' \
-              -v $CODE_DIR:$CODE_DIR_IN_CONTAINER \
+              -v "$CODE_DIR:$SOURCE_DIR_IN_CONTAINER:ro" \
               -v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
               -v $MODELSCOPE_HOME_CACHE/$idx:/root \
               -v /home/admin/pre-commit:/home/admin/pre-commit \
@@ -75,7 +77,8 @@ do
 	            -e PR_CHANGED_FILES=$PR_CHANGED_FILES \
               --workdir=$CODE_DIR_IN_CONTAINER \
               ${IMAGE_NAME}:${IMAGE_VERSION} \
-              $CI_COMMAND
+              bash "$SOURCE_DIR_IN_CONTAINER/.dev_scripts/ci_source_copy.sh" \
+              "$SOURCE_DIR_IN_CONTAINER" $CI_COMMAND
   fi
   if [ $? -ne 0 ]; then
     echo "Running test case failed, please check the log!"

.dev_scripts/test_ci_source_copy.py

@@ -0,0 +1,86 @@
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPT = Path(__file__).with_name('ci_source_copy.sh')
+
+
+class SourceCopyTest(unittest.TestCase):
+
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        self.addCleanup(self.directory.cleanup)
+        self.source = Path(self.directory.name) / 'source with spaces'
+        self.source.mkdir()
+        (self.source / 'input.txt').write_text('original')
+        (self.source / '.git').mkdir()
+        (self.source / '.git/config').write_text('original git configuration')
+        (self.source / 'linked.txt').symlink_to('input.txt')
+
+    def run_copy(self, *command, source=None):
+        return subprocess.run(['bash', str(SCRIPT), str(source or self.source), *command],
+                              capture_output=True,
+                              text=True,
+                              check=False)
+
+    def test_private_copy_preserves_inputs_and_command_arguments(self):
+        result = self.run_copy(
+            sys.executable, '-c', '''
+import json
+import sys
+from pathlib import Path
+
+cwd = Path.cwd()
+assert (cwd / 'input.txt').read_text() == 'original'
+assert (cwd / 'linked.txt').is_symlink()
+assert (cwd / 'linked.txt').read_text() == 'original'
+(cwd / 'input.txt').write_text('modified')
+(cwd / '.git/config').write_text('modified git configuration')
+(cwd / 'build-output').write_text('output')
+print(json.dumps({'cwd': str(cwd), 'args': sys.argv[1:]}))
+''', 'argument with spaces', '*')
+        self.assertEqual(result.returncode, 0, result.stderr)
+        report = json.loads(result.stdout)
+        self.assertEqual(report['args'], ['argument with spaces', '*'])
+        self.assertNotEqual(Path(report['cwd']), self.source)
+        self.assertFalse(Path(report['cwd']).exists())
+        self.assertEqual((self.source / 'input.txt').read_text(), 'original')
+        self.assertEqual((self.source / '.git/config').read_text(), 'original git configuration')
+        self.assertFalse((self.source / 'build-output').exists())
+
+    def test_failure_is_preserved_and_private_copy_is_removed(self):
+        result = self.run_copy(
+            sys.executable, '-c', '''
+import sys
+from pathlib import Path
+
+print(Path.cwd())
+Path('build-output').write_text('partial output')
+sys.exit(23)
+''')
+        self.assertEqual(result.returncode, 23, result.stderr)
+        self.assertFalse(Path(result.stdout.strip()).exists())
+        self.assertFalse((self.source / 'build-output').exists())
+
+    def test_copy_failure_prevents_command_execution(self):
+        marker = Path(self.directory.name) / 'command-ran'
+        result = self.run_copy(
+            sys.executable,
+            '-c',
+            'from pathlib import Path; import sys; Path(sys.argv[1]).touch()',
+            str(marker),
+            source=self.source / 'missing')
+        self.assertNotEqual(result.returncode, 0)
+        self.assertFalse(marker.exists())
+
+    def test_missing_command_is_rejected(self):
+        result = self.run_copy()
+        self.assertEqual(result.returncode, 2)
+        self.assertIn('Usage:', result.stderr)
+
+
+if __name__ == '__main__':
+    unittest.main()

.github/workflows/alignment_model_accuracy.yaml

@@ -37,7 +37,7 @@ jobs:
           determine_gpu_runner ${runner_name}
       - name: Determine Image Name
         run: |
-          echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda130-py312-dev" >> "$GITHUB_ENV"
+          echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev" >> "$GITHUB_ENV"
       - name: Run Container
         run: |
           container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
@@ -65,7 +65,7 @@ jobs:
             wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleFleet.tar --no-check-certificate
             rm -rf PaddleFleet && tar xf PaddleFleet.tar && rm -rf PaddleFleet.tar
             cd PaddleFleet && git pull && cd -
-            
+
             echo "Download ms-swift form https://paddle-github-action.bj.bcebos.com/whl/ms-swift.tar.gz"
             wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/whl/ms-swift.tar.gz --no-check-certificate
             rm -rf ms-swift && tar zxf ms-swift.tar.gz && rm -rf ms-swift.tar.gz
@@ -114,7 +114,7 @@ jobs:
           cd /workspace
           for url in \
             $BOS/PaddleFleet/develop/latest/paddlefleet-0.0.0-py3-none-linux_x86_64.whl \
-            $BOS/PaddleFleet/develop/latest/cu130/paddle-release/paddlefleet_ops-0.0.0-cp312-cp312-linux_x86_64.whl \
+            $BOS/PaddleFleet/develop/latest/cu129/paddle-release/paddlefleet_ops-0.0.0-cp312-cp312-linux_x86_64.whl \
             $BOS/whl/megatron_core-0.0.0-cp312-cp312-linux_x86_64.whl \
             $BOS/whl/mcore_bridge-0.0.0-py3-none-any.whl ; do
             echo "Downloading $url"
@@ -145,7 +145,7 @@ jobs:
           export MEGATRON_CORE_WHEEL_PATH=/workspace/megatron_core-0.0.0-cp312-cp312-linux_x86_64.whl
           export MS_SWIFT_WHEEL_PATH=/workspace/upload/ms_swift-0.0.0-py3-none-any.whl
           export MCORE_BRIDGE_WHEEL_PATH=/workspace/mcore_bridge-0.0.0-py3-none-any.whl
-          # The BOS wheels are republished under a fixed 0.0.0 filename 
+          # The BOS wheels are republished under a fixed 0.0.0 filename
           export UV_SKIP_WHEEL_FILENAME_CHECK=1
 
           for whl in "$PADDLEFLEET_WHEEL_PATH" "$PADDLEFLEET_OPS_WHEEL_PATH" \

.github/workflows/citest.yaml

@@ -11,7 +11,6 @@ on:
       - "requirements/**"
       - "docs/**"
       - "tools/**"
-      - ".dev_scripts/**"
       - "README.md"
       - "README_*.md"
       - "NOTICE"
@@ -25,7 +24,6 @@ on:
       - "requirements/**"
       - "docs/**"
       - "tools/**"
-      - ".dev_scripts/**"
       - "README.md"
       - "README_*.md"
       - "NOTICE"
@@ -41,37 +39,73 @@ jobs:
     # The type of runner that the job will run on
     runs-on: [self-hosted]
     timeout-minutes: 240
+    env:
+      CI_CHECKOUT_PATH: .citest-${{ github.run_id }}-${{ github.run_attempt }}
     steps:
-      - name: ResetFileMode
-        shell: bash
-        run: |
-          # reset filemode to allow action runner to delete files
-          # generated by root in docker
-          set -e
-          source ~/.bashrc
-          sudo chown -R $USER:$USER  $GITHUB_WORKSPACE
-
       - name: Checkout
-        uses: actions/checkout@v3
+        shell: bash
         env:
           GIT_CONFIG_PARAMETERS: "'core.hooksPath='"
-        with:
-            lfs: 'true'
-            submodules: 'false'
-            fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}
+        run: |
+          set -euo pipefail
+          mkdir -- "$CI_CHECKOUT_PATH"
+          cd "$CI_CHECKOUT_PATH"
+          git init
+          git remote add origin "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git"
+          fetch_options=()
+          if [[ "$GITHUB_EVENT_NAME" == pull_request ]]; then
+            fetch_options+=(--depth=2)
+          fi
+          git fetch --no-tags "${fetch_options[@]}" origin "$GITHUB_SHA"
+          GIT_LFS_SKIP_SMUDGE=1 git checkout --detach "$GITHUB_SHA"
       - name: Get changed files
         id: changed-files
+        working-directory: ${{ env.CI_CHECKOUT_PATH }}
         run: |
             if ${{ github.event_name == 'pull_request' }}; then
-                echo "PR_CHANGED_FILES=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_ENV
+                echo "PR_CHANGED_FILES=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> "$GITHUB_ENV"
             else
-                echo "PR_CHANGED_FILES=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | xargs)" >> $GITHUB_ENV
+                echo "PR_CHANGED_FILES=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | xargs)" >> "$GITHUB_ENV"
             fi
       - name: Checkout LFS objects
-        run: git lfs checkout
+        working-directory: ${{ env.CI_CHECKOUT_PATH }}
+        run: |
+          set -euo pipefail
+          if ! git lfs version; then
+            case "$(uname -m)" in
+              x86_64)
+                architecture=amd64
+                digest=e455e00f15d9b95661b8d53498ffb0c3367962cf1ec73c31ab7369516cd6ab8d
+                ;;
+              aarch64)
+                architecture=arm64
+                digest=ac9c8efac980bb0505ead384d087e2acb6486fd8498691a2165fa174ec6118c2
+                ;;
+              *) echo "Unsupported Git LFS host architecture" >&2; exit 1 ;;
+            esac
+            tool_dir="$(mktemp -d "${RUNNER_TEMP:?}/git-lfs.XXXXXX")"
+            trap 'rm -rf -- "${tool_dir:?}"' EXIT
+            archive="$tool_dir/git-lfs.tar.gz"
+            curl --fail --location --retry 3 \
+              "https://github.com/git-lfs/git-lfs/releases/download/v3.8.0/git-lfs-linux-$architecture-v3.8.0.tar.gz" \
+              --output "$archive"
+            echo "$digest  $archive" | sha256sum --check -
+            tar -xzf "$archive" -C "$tool_dir" --strip-components=1 git-lfs-3.8.0/git-lfs
+            export PATH="$tool_dir:$PATH"
+            git lfs version
+          fi
+          git lfs install --local --skip-smudge --skip-repo
+          git lfs fetch origin "$GITHUB_SHA"
+          git lfs checkout
+          git lfs fsck
       - name: Run unittest
         shell: bash
+        working-directory: ${{ env.CI_CHECKOUT_PATH }}
         run: |
           set -e
           source /mnt/modelscope/ci_env.sh
           bash .dev_scripts/dockerci.sh
+      - name: Remove this run's checkout
+        if: always()
+        shell: bash
+        run: rm -rf -- "${GITHUB_WORKSPACE:?}/${CI_CHECKOUT_PATH:?}"

.github/workflows/lint.yaml

@@ -10,13 +10,15 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
       - name: Set up Python 3.10
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
         with:
           python-version: '3.10'
       - name: Install pre-commit hook
         run: |
           pip install pre-commit
       - name: Linting
         run: pre-commit run --all-files
+      - name: CI source isolation regression tests
+        run: python .dev_scripts/test_ci_source_copy.py

scripts/dependence/build.sh

@@ -79,4 +79,3 @@ echo -e "\033[32m ---- make ms-swift.tar.gz  \033[0m"
 swift_tar
 echo -e "\033[32m ---- build ms-swift whl  \033[0m"
 swift_build
-

swift/megatron/init.py

@@ -10,10 +10,10 @@
 from copy import copy, deepcopy
 from packaging import version
 from tqdm import tqdm
-from typing import Optional
 from transformers.modeling_utils import custom_object_save
 from transformers.utils import is_torch_npu_available
 from transformers.utils.versions import require_version
+from typing import Optional
 
 from swift.model import get_model_processor, save_checkpoint
 from swift.utils import (HfConfigFactory, disable_safe_ddp_context_use_barrier, get_logger, get_modules_to_not_convert,
@@ -178,6 +178,7 @@ def _patch_mcore_bridge_disable_te():
     import mcore_bridge.model.register as mcb_register
 
     def _force_local_spec(orig):
+
         def wrapper(*args, **kwargs):
             kwargs['use_transformer_engine'] = False
             return orig(*args, **kwargs)

swift/megatron/pipelines/train/sft.py

@@ -43,9 +43,6 @@ def __init__(self, args: Optional[Union[List[str], MegatronSftArguments]] = None
         self.train_msg = {}
         super(SwiftSft, self).__init__(args)
         args = self.args
-        from megatron.core.transformer.module import _use_accuracy_compatible
-        if _use_accuracy_compatible() and getattr(args, 'clip_grad', 0) > 0:
-            args.clip_grad = 0.0
         if repatch is not None:
             megatron_args = asdict(self.args)
             if args.attention_backend != 'local':

swift/megatron/trainers/base.py

@@ -207,6 +207,10 @@ def _prepare_peft_model(self, models):
 
     def get_optimizer_and_scheduler(self):
         args = self.args
+        if args.use_accuracy_compatible and args.clip_grad > 0:
+            from megatron.core.optimizer import clip_grads
+            if not hasattr(clip_grads, 'get_reproducible_grad_norm_bins'):
+                raise ValueError('Accuracy-compatible clipping requires Megatron-Core reproducible norm support')
         if mcore_016:
             from megatron.core.optimizer import AdamOptimizerConfig, SGDOptimizerConfig
             if args.optimizer == 'adam' or 'muon' in args.optimizer:

swift/megatron/trainers/trainer.py

@@ -83,9 +83,9 @@ def loss_func(self,
             import hashlib as _hashlib
             _final = (loss[0].detach().float() / loss[1].detach().float().clamp(min=1)).contiguous()
             print(
-                f"\nfinal_loss: rank={torch.distributed.get_rank()} "
-                f"val={_final.item():.20f} "
-                f"md5={_hashlib.md5(_final.cpu().numpy().tobytes()).hexdigest()}",
+                f'\nfinal_loss: rank={torch.distributed.get_rank()} '
+                f'val={_final.item():.20f} '
+                f'md5={_hashlib.md5(_final.cpu().numpy().tobytes()).hexdigest()}',
                 flush=True)
 
         metrics = {'loss': reporting_loss}