diff --git a/.dev_scripts/ci_source_copy.sh b/.dev_scripts/ci_source_copy.sh new file mode 100644 index 0000000000..216e2ebdf2 --- /dev/null +++ b/.dev_scripts/ci_source_copy.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +if [ "$#" -lt 2 ]; then + echo "Usage: $0 SOURCE_DIR COMMAND [ARG...]" >&2 + exit 2 +fi + +source_dir=$1 +shift +test_dir="$(mktemp -d /tmp/swift-ci.XXXXXX)" +trap 'rm -rf -- "${test_dir:?}"' EXIT + +# Build products and Git changes stay in the container, owned by its test user. +cp -R "$source_dir/." "$test_dir/" +cd "$test_dir" +"$@" diff --git a/.dev_scripts/dockerci.sh b/.dev_scripts/dockerci.sh index f42e819fb4..099d1c7fd9 100644 --- a/.dev_scripts/dockerci.sh +++ b/.dev_scripts/dockerci.sh @@ -2,6 +2,7 @@ MODELSCOPE_CACHE_DIR_IN_CONTAINER=/modelscope_cache CODE_DIR=$PWD CODE_DIR_IN_CONTAINER=/ms-swift +SOURCE_DIR_IN_CONTAINER=/ms-swift-source MODELSCOPE_SDK_DEBUG=True echo "$USER" gpus='0,1 2,3' @@ -31,7 +32,7 @@ do docker run --rm --name $CONTAINER_NAME --shm-size=16gb \ --cpuset-cpus=${cpu_sets_arr[$idx]} \ --gpus='"'"device=$gpu"'"' \ - -v $CODE_DIR:$CODE_DIR_IN_CONTAINER \ + -v "$CODE_DIR:$SOURCE_DIR_IN_CONTAINER:ro" \ -v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \ -v $MODELSCOPE_HOME_CACHE/$idx:/root \ -v /home/admin/pre-commit:/home/admin/pre-commit \ @@ -51,12 +52,13 @@ do -e PR_CHANGED_FILES=$PR_CHANGED_FILES \ --workdir=$CODE_DIR_IN_CONTAINER \ ${IMAGE_NAME}:${IMAGE_VERSION} \ - $CI_COMMAND + bash "$SOURCE_DIR_IN_CONTAINER/.dev_scripts/ci_source_copy.sh" \ + "$SOURCE_DIR_IN_CONTAINER" $CI_COMMAND else docker run --rm --name $CONTAINER_NAME --shm-size=16gb \ --cpuset-cpus=${cpu_sets_arr[$idx]} \ --gpus='"'"device=$gpu"'"' \ - -v $CODE_DIR:$CODE_DIR_IN_CONTAINER \ + -v "$CODE_DIR:$SOURCE_DIR_IN_CONTAINER:ro" \ -v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \ -v $MODELSCOPE_HOME_CACHE/$idx:/root \ -v /home/admin/pre-commit:/home/admin/pre-commit \ @@ -75,7 +77,8 @@ do -e PR_CHANGED_FILES=$PR_CHANGED_FILES \ --workdir=$CODE_DIR_IN_CONTAINER \ ${IMAGE_NAME}:${IMAGE_VERSION} \ - $CI_COMMAND + bash "$SOURCE_DIR_IN_CONTAINER/.dev_scripts/ci_source_copy.sh" \ + "$SOURCE_DIR_IN_CONTAINER" $CI_COMMAND fi if [ $? -ne 0 ]; then echo "Running test case failed, please check the log!" diff --git a/.dev_scripts/test_ci_source_copy.py b/.dev_scripts/test_ci_source_copy.py new file mode 100644 index 0000000000..87db867698 --- /dev/null +++ b/.dev_scripts/test_ci_source_copy.py @@ -0,0 +1,86 @@ +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).with_name('ci_source_copy.sh') + + +class SourceCopyTest(unittest.TestCase): + + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.source = Path(self.directory.name) / 'source with spaces' + self.source.mkdir() + (self.source / 'input.txt').write_text('original') + (self.source / '.git').mkdir() + (self.source / '.git/config').write_text('original git configuration') + (self.source / 'linked.txt').symlink_to('input.txt') + + def run_copy(self, *command, source=None): + return subprocess.run(['bash', str(SCRIPT), str(source or self.source), *command], + capture_output=True, + text=True, + check=False) + + def test_private_copy_preserves_inputs_and_command_arguments(self): + result = self.run_copy( + sys.executable, '-c', ''' +import json +import sys +from pathlib import Path + +cwd = Path.cwd() +assert (cwd / 'input.txt').read_text() == 'original' +assert (cwd / 'linked.txt').is_symlink() +assert (cwd / 'linked.txt').read_text() == 'original' +(cwd / 'input.txt').write_text('modified') +(cwd / '.git/config').write_text('modified git configuration') +(cwd / 'build-output').write_text('output') +print(json.dumps({'cwd': str(cwd), 'args': sys.argv[1:]})) +''', 'argument with spaces', '*') + self.assertEqual(result.returncode, 0, result.stderr) + report = json.loads(result.stdout) + self.assertEqual(report['args'], ['argument with spaces', '*']) + self.assertNotEqual(Path(report['cwd']), self.source) + self.assertFalse(Path(report['cwd']).exists()) + self.assertEqual((self.source / 'input.txt').read_text(), 'original') + self.assertEqual((self.source / '.git/config').read_text(), 'original git configuration') + self.assertFalse((self.source / 'build-output').exists()) + + def test_failure_is_preserved_and_private_copy_is_removed(self): + result = self.run_copy( + sys.executable, '-c', ''' +import sys +from pathlib import Path + +print(Path.cwd()) +Path('build-output').write_text('partial output') +sys.exit(23) +''') + self.assertEqual(result.returncode, 23, result.stderr) + self.assertFalse(Path(result.stdout.strip()).exists()) + self.assertFalse((self.source / 'build-output').exists()) + + def test_copy_failure_prevents_command_execution(self): + marker = Path(self.directory.name) / 'command-ran' + result = self.run_copy( + sys.executable, + '-c', + 'from pathlib import Path; import sys; Path(sys.argv[1]).touch()', + str(marker), + source=self.source / 'missing') + self.assertNotEqual(result.returncode, 0) + self.assertFalse(marker.exists()) + + def test_missing_command_is_rejected(self): + result = self.run_copy() + self.assertEqual(result.returncode, 2) + self.assertIn('Usage:', result.stderr) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/alignment_model_accuracy.yaml b/.github/workflows/alignment_model_accuracy.yaml index 76faf444ba..24d16e98a9 100644 --- a/.github/workflows/alignment_model_accuracy.yaml +++ b/.github/workflows/alignment_model_accuracy.yaml @@ -37,7 +37,7 @@ jobs: determine_gpu_runner ${runner_name} - name: Determine Image Name run: | - echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda130-py312-dev" >> "$GITHUB_ENV" + echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev" >> "$GITHUB_ENV" - name: Run Container run: | container_name=${TASK}-$(date +%Y%m%d-%H%M%S) @@ -65,7 +65,7 @@ jobs: wget -q --no-proxy https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleFleet.tar --no-check-certificate rm -rf PaddleFleet && tar xf PaddleFleet.tar && rm -rf PaddleFleet.tar cd PaddleFleet && git pull && cd - - + echo "Download ms-swift form https://paddle-github-action.bj.bcebos.com/whl/ms-swift.tar.gz" wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/whl/ms-swift.tar.gz --no-check-certificate rm -rf ms-swift && tar zxf ms-swift.tar.gz && rm -rf ms-swift.tar.gz @@ -114,7 +114,7 @@ jobs: cd /workspace for url in \ $BOS/PaddleFleet/develop/latest/paddlefleet-0.0.0-py3-none-linux_x86_64.whl \ - $BOS/PaddleFleet/develop/latest/cu130/paddle-release/paddlefleet_ops-0.0.0-cp312-cp312-linux_x86_64.whl \ + $BOS/PaddleFleet/develop/latest/cu129/paddle-release/paddlefleet_ops-0.0.0-cp312-cp312-linux_x86_64.whl \ $BOS/whl/megatron_core-0.0.0-cp312-cp312-linux_x86_64.whl \ $BOS/whl/mcore_bridge-0.0.0-py3-none-any.whl ; do echo "Downloading $url" @@ -145,7 +145,7 @@ jobs: export MEGATRON_CORE_WHEEL_PATH=/workspace/megatron_core-0.0.0-cp312-cp312-linux_x86_64.whl export MS_SWIFT_WHEEL_PATH=/workspace/upload/ms_swift-0.0.0-py3-none-any.whl export MCORE_BRIDGE_WHEEL_PATH=/workspace/mcore_bridge-0.0.0-py3-none-any.whl - # The BOS wheels are republished under a fixed 0.0.0 filename + # The BOS wheels are republished under a fixed 0.0.0 filename export UV_SKIP_WHEEL_FILENAME_CHECK=1 for whl in "$PADDLEFLEET_WHEEL_PATH" "$PADDLEFLEET_OPS_WHEEL_PATH" \ @@ -155,7 +155,6 @@ jobs: echo "using $whl" done python -m pip install uv - bash PaddleFleet/scripts/alignment_model_accuracy/setup_venvs.sh bash -x PaddleFleet/scripts/alignment_model_accuracy/run_alignment_test.sh model_acc_align_exit_code=$? diff --git a/.github/workflows/citest.yaml b/.github/workflows/citest.yaml index 101e9f3b1a..575dc01efa 100644 --- a/.github/workflows/citest.yaml +++ b/.github/workflows/citest.yaml @@ -11,7 +11,6 @@ on: - "requirements/**" - "docs/**" - "tools/**" - - ".dev_scripts/**" - "README.md" - "README_*.md" - "NOTICE" @@ -25,7 +24,6 @@ on: - "requirements/**" - "docs/**" - "tools/**" - - ".dev_scripts/**" - "README.md" - "README_*.md" - "NOTICE" @@ -41,37 +39,82 @@ jobs: # The type of runner that the job will run on runs-on: [self-hosted] timeout-minutes: 240 + env: + CI_CHECKOUT_PATH: .citest-${{ github.run_id }}-${{ github.run_attempt }} steps: - - name: ResetFileMode - shell: bash - run: | - # reset filemode to allow action runner to delete files - # generated by root in docker - set -e - source ~/.bashrc - sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - name: Checkout - uses: actions/checkout@v3 + shell: bash env: GIT_CONFIG_PARAMETERS: "'core.hooksPath='" - with: - lfs: 'true' - submodules: 'false' - fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} + run: | + set -euo pipefail + mkdir -- "$CI_CHECKOUT_PATH" + cd "$CI_CHECKOUT_PATH" + git init + git remote add origin "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" + fetch_options=() + if [[ "$GITHUB_EVENT_NAME" == pull_request ]]; then + fetch_options+=(--depth=2) + fi + git fetch --no-tags "${fetch_options[@]}" origin "$GITHUB_SHA" + GIT_LFS_SKIP_SMUDGE=1 git checkout --detach "$GITHUB_SHA" - name: Get changed files id: changed-files + working-directory: ${{ env.CI_CHECKOUT_PATH }} run: | if ${{ github.event_name == 'pull_request' }}; then - echo "PR_CHANGED_FILES=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_ENV + echo "PR_CHANGED_FILES=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> "$GITHUB_ENV" else - echo "PR_CHANGED_FILES=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | xargs)" >> $GITHUB_ENV + echo "PR_CHANGED_FILES=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | xargs)" >> "$GITHUB_ENV" fi - name: Checkout LFS objects - run: git lfs checkout + working-directory: ${{ env.CI_CHECKOUT_PATH }} + run: | + set -euo pipefail + # Revisions without LFS pointers need no hydration or Git LFS installation. + if git grep -I -q '^version https://git-lfs.github.com/spec/v1$' HEAD --; then + echo "Hydrating this revision's LFS objects" + else + status=$? + if [ "$status" -ne 1 ]; then exit "$status"; fi + echo "No LFS pointers in this revision" + exit 0 + fi + if ! git lfs version; then + case "$(uname -m)" in + x86_64) + architecture=amd64 + digest=e455e00f15d9b95661b8d53498ffb0c3367962cf1ec73c31ab7369516cd6ab8d + ;; + aarch64) + architecture=arm64 + digest=ac9c8efac980bb0505ead384d087e2acb6486fd8498691a2165fa174ec6118c2 + ;; + *) echo "Unsupported Git LFS host architecture" >&2; exit 1 ;; + esac + tool_dir="$(mktemp -d "${RUNNER_TEMP:?}/git-lfs.XXXXXX")" + trap 'rm -rf -- "${tool_dir:?}"' EXIT + archive="$tool_dir/git-lfs.tar.gz" + curl --fail --location --retry 3 \ + "https://github.com/git-lfs/git-lfs/releases/download/v3.8.0/git-lfs-linux-$architecture-v3.8.0.tar.gz" \ + --output "$archive" + echo "$digest $archive" | sha256sum --check - + tar -xzf "$archive" -C "$tool_dir" --strip-components=1 git-lfs-3.8.0/git-lfs + export PATH="$tool_dir:$PATH" + git lfs version + fi + git lfs install --local --skip-smudge --skip-repo + git lfs fetch origin "$GITHUB_SHA" + git lfs checkout + git lfs fsck - name: Run unittest shell: bash + working-directory: ${{ env.CI_CHECKOUT_PATH }} run: | set -e source /mnt/modelscope/ci_env.sh bash .dev_scripts/dockerci.sh + - name: Remove this run's checkout + if: always() + shell: bash + run: rm -rf -- "${GITHUB_WORKSPACE:?}/${CI_CHECKOUT_PATH:?}" diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 6ff84517d8..8ae4a2ec6f 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -10,9 +10,9 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Set up Python 3.10 - uses: actions/setup-python@v2 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: '3.10' - name: Install pre-commit hook @@ -20,3 +20,5 @@ jobs: pip install pre-commit - name: Linting run: pre-commit run --all-files + - name: CI source isolation regression tests + run: python .dev_scripts/test_ci_source_copy.py diff --git a/scripts/dependence/build.sh b/scripts/dependence/build.sh index de73702692..6b9129513c 100644 --- a/scripts/dependence/build.sh +++ b/scripts/dependence/build.sh @@ -79,4 +79,3 @@ echo -e "\033[32m ---- make ms-swift.tar.gz \033[0m" swift_tar echo -e "\033[32m ---- build ms-swift whl \033[0m" swift_build - diff --git a/swift/megatron/init.py b/swift/megatron/init.py index bcfda11d7a..5d2ee29f78 100644 --- a/swift/megatron/init.py +++ b/swift/megatron/init.py @@ -10,10 +10,10 @@ from copy import copy, deepcopy from packaging import version from tqdm import tqdm -from typing import Optional from transformers.modeling_utils import custom_object_save from transformers.utils import is_torch_npu_available from transformers.utils.versions import require_version +from typing import Optional from swift.model import get_model_processor, save_checkpoint from swift.utils import (HfConfigFactory, disable_safe_ddp_context_use_barrier, get_logger, get_modules_to_not_convert, @@ -178,6 +178,7 @@ def _patch_mcore_bridge_disable_te(): import mcore_bridge.model.register as mcb_register def _force_local_spec(orig): + def wrapper(*args, **kwargs): kwargs['use_transformer_engine'] = False return orig(*args, **kwargs) diff --git a/swift/megatron/trainers/trainer.py b/swift/megatron/trainers/trainer.py index 46fa9914d4..5f9174db05 100644 --- a/swift/megatron/trainers/trainer.py +++ b/swift/megatron/trainers/trainer.py @@ -83,9 +83,9 @@ def loss_func(self, import hashlib as _hashlib _final = (loss[0].detach().float() / loss[1].detach().float().clamp(min=1)).contiguous() print( - f"\nfinal_loss: rank={torch.distributed.get_rank()} " - f"val={_final.item():.20f} " - f"md5={_hashlib.md5(_final.cpu().numpy().tobytes()).hexdigest()}", + f'\nfinal_loss: rank={torch.distributed.get_rank()} ' + f'val={_final.item():.20f} ' + f'md5={_hashlib.md5(_final.cpu().numpy().tobytes()).hexdigest()}', flush=True) metrics = {'loss': reporting_loss} diff --git a/swift/megatron/utils/utils.py b/swift/megatron/utils/utils.py index 9562475849..b4c43efcf5 100644 --- a/swift/megatron/utils/utils.py +++ b/swift/megatron/utils/utils.py @@ -289,7 +289,7 @@ def get_load_fixed_data_path(): def _batch_data_suffix(step, rank, seq_len): - return f"step{step}_rank{rank}_seq{seq_len}.npy" + return f'step{step}_rank{rank}_seq{seq_len}.npy' def dump_batch_data(batch, step, seq_len): @@ -304,10 +304,10 @@ def dump_batch_data(batch, step, seq_len): torch.cuda.synchronize() os.makedirs(dump_path, exist_ok=True) suffix = _batch_data_suffix(step, rank, seq_len) - np.save(os.path.join(dump_path, f"tokens_{suffix}"), tokens.detach().cpu().numpy()) - np.save(os.path.join(dump_path, f"labels_{suffix}"), labels.detach().cpu().numpy()) + np.save(os.path.join(dump_path, f'tokens_{suffix}'), tokens.detach().cpu().numpy()) + np.save(os.path.join(dump_path, f'labels_{suffix}'), labels.detach().cpu().numpy()) if rank == 0: - print(f"[DUMP_DATA_PATH] saved tokens_{suffix} and labels_{suffix}", flush=True) + print(f'[DUMP_DATA_PATH] saved tokens_{suffix} and labels_{suffix}', flush=True) def load_fixed_batch_data(batch, step, seq_len): @@ -317,11 +317,11 @@ def load_fixed_batch_data(batch, step, seq_len): rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0 suffix = _batch_data_suffix(step, rank, seq_len) - tokens_file = os.path.join(load_path, f"tokens_{suffix}") - labels_file = os.path.join(load_path, f"labels_{suffix}") + tokens_file = os.path.join(load_path, f'tokens_{suffix}') + labels_file = os.path.join(load_path, f'labels_{suffix}') if not (os.path.exists(tokens_file) and os.path.exists(labels_file)): if rank == 0: - print(f"[LOAD_FIXED_DATA_PATH] file not found: {tokens_file}", flush=True) + print(f'[LOAD_FIXED_DATA_PATH] file not found: {tokens_file}', flush=True) return batch tokens_np = np.load(tokens_file)