Aiter Test #197
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Aiter Test | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| branches: [main] # Triggers on PRs targeting `main` | |
| paths: | |
| - '**' | |
| - '!**/*.md' | |
| - '!docs/**' | |
| - '!LICENSE' | |
| - '!.gitignore' | |
| - '!.github/workflows/**' | |
| - '.github/workflows/aiter-test.yaml' | |
| - '!.github/scripts/sglang_downstream.py' | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 22 * * *' # 6:00 AM Beijing Time (UTC+8) | |
| concurrency: | |
| # Keep scheduled main runs from blocking push-triggered validation. | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| env: | |
| DOCKER_IMAGE: "rocm/pytorch:latest" | |
| GPU_ARCH_LIST: "gfx942;gfx950" | |
| AITER_TEST: "op_tests" | |
| AITER_WHEEL_ARTIFACT_NAME: aiter-whl-${{ github.run_id }} | |
| AITER_PREBUILD_MAX_JOBS: "64" | |
| TRITON_WHEEL_ARTIFACT_NAME: triton_wheelhouse | |
| jobs: | |
| check-signal: | |
| if: ${{ !github.event.pull_request || github.event.pull_request.draft == false }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Wait for Checks workflow | |
| run: ./.github/scripts/check_signal.sh | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_SHA: ${{ github.sha }} | |
| build_aiter_wheels: | |
| if: ${{ !github.event.pull_request || github.event.pull_request.draft == false }} | |
| runs-on: build-only-aiter | |
| needs: check-signal | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - python_version: "3.10" | |
| docker_image: "rocm/pytorch:rocm7.2.4_ubuntu22.04_py3.10_pytorch_release_2.10.0" | |
| build_enabled: ${{ github.ref == 'refs/heads/main' }} | |
| - python_version: "3.12" | |
| docker_image: "rocm/pytorch:latest" | |
| build_enabled: true | |
| steps: | |
| # ---- Common steps (fork and non-fork) ---- | |
| - name: Checkout code | |
| if: ${{ matrix.build_enabled }} | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - name: Sync submodules for wheel build | |
| if: ${{ matrix.build_enabled }} | |
| run: | | |
| set -ex | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| echo "Nightly build: syncing latest CK from develop branch..." | |
| git submodule set-branch --branch develop 3rdparty/composable_kernel | |
| git submodule sync | |
| git submodule update --init --recursive --remote --jobs 4 | |
| else | |
| echo "Using pinned CK commit..." | |
| git submodule sync | |
| git submodule update --init --recursive --depth 1 --jobs 4 | |
| fi | |
| echo "CK commit: $(git -C 3rdparty/composable_kernel rev-parse HEAD)" | |
| - name: Docker login | |
| if: ${{ matrix.build_enabled && (!github.event.pull_request || !github.event.pull_request.head.repo.fork) }} | |
| env: | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} | |
| run: | | |
| for attempt in 1 2 3; do | |
| if echo "$DOCKER_PASSWORD" | docker login -u rocmshared --password-stdin; then | |
| echo "Docker login succeeded on attempt ${attempt}" | |
| exit 0 | |
| fi | |
| echo "Docker login attempt ${attempt} failed" | |
| if [ "${attempt}" != 3 ]; then | |
| sleep 10 | |
| fi | |
| done | |
| echo "Docker login failed after 3 attempts, continuing anyway" | |
| exit 0 | |
| - name: Build Aiter wheel in base container | |
| if: ${{ matrix.build_enabled }} | |
| run: | | |
| set -euo pipefail | |
| docker run --rm \ | |
| --network=host \ | |
| -e AITER_PREBUILD_MAX_JOBS="${{ env.AITER_PREBUILD_MAX_JOBS }}" \ | |
| -e AITER_RUNNER_NAME="${RUNNER_NAME:-unknown}" \ | |
| -v "${{ github.workspace }}:/workspace" \ | |
| -w /workspace \ | |
| ${{ matrix.docker_image }} \ | |
| bash -lc ' | |
| set -euo pipefail | |
| git config --global --add safe.directory /workspace && | |
| git -C /workspace rev-parse HEAD >/dev/null && | |
| shopt -s nullglob && | |
| rm -rf dist build aiter_meta ./*.egg-info && | |
| pip install -r requirements.txt && | |
| pip install --upgrade pandas pyzmq einops numpy==1.26.2 && | |
| pip install --upgrade "pybind11>=3.0.1" && | |
| pip install --upgrade "ninja>=1.11.1" && | |
| pip install --upgrade setuptools_scm && | |
| pip install tabulate && | |
| echo "Prebuilding kernels with GPU_ARCHS: ${{ env.GPU_ARCH_LIST }}, PREBUILD_KERNELS: 1, and MAX_JOBS: ${AITER_PREBUILD_MAX_JOBS}" && | |
| export PREBUILD_KERNELS=1 && | |
| export MAX_JOBS="${AITER_PREBUILD_MAX_JOBS}" && | |
| export GPU_ARCHS="${{ env.GPU_ARCH_LIST }}" && | |
| prebuild_start=$(date +%s) && | |
| set +e && | |
| python setup.py bdist_wheel 2>&1 | tee .aiter-prebuild.log | |
| build_status=${PIPESTATUS[0]} | |
| set -e | |
| prebuild_end=$(date +%s) | |
| { | |
| echo "BUILD_STATUS=${build_status}" | |
| echo "PREBUILD_START=${prebuild_start}" | |
| echo "PREBUILD_END=${prebuild_end}" | |
| } > .aiter-prebuild.env | |
| if [ "${build_status}" -ne 0 ]; then | |
| exit "${build_status}" | |
| fi | |
| ls -lh dist/*.whl | |
| ' | |
| - name: Summarize Aiter prebuild | |
| if: ${{ always() && matrix.build_enabled }} | |
| run: | | |
| set -euo pipefail | |
| if [ ! -f .aiter-prebuild.env ]; then | |
| echo "::warning::Aiter prebuild metadata was not generated" | |
| exit 0 | |
| fi | |
| . ./.aiter-prebuild.env | |
| AITER_RUNNER_NAME="${RUNNER_NAME:-unknown}" \ | |
| GPU_ARCHS="${{ env.GPU_ARCH_LIST }}" \ | |
| PREBUILD_KERNELS=1 \ | |
| MAX_JOBS="${{ env.AITER_PREBUILD_MAX_JOBS }}" \ | |
| python3 .github/scripts/aiter_prebuild_summary.py \ | |
| --log .aiter-prebuild.log \ | |
| --build-status "${BUILD_STATUS}" \ | |
| --start "${PREBUILD_START}" \ | |
| --end "${PREBUILD_END}" | |
| - name: Verify prebuilt kernels in wheel | |
| if: ${{ matrix.build_enabled }} | |
| run: | | |
| set -euo pipefail | |
| python3 - <<'PY' | |
| import glob | |
| import zipfile | |
| wheels = glob.glob("dist/*.whl") | |
| if len(wheels) != 1: | |
| raise SystemExit(f"Expected exactly one wheel in dist/, found {len(wheels)}") | |
| wheel = wheels[0] | |
| with zipfile.ZipFile(wheel) as zf: | |
| kernels = sorted( | |
| name for name in zf.namelist() | |
| if name.startswith("aiter/jit/") and name.endswith(".so") | |
| ) | |
| print("=== Prebuilt kernel validation ===") | |
| print(f"Wheel: {wheel}") | |
| print(f"Prebuilt kernel .so files: {len(kernels)}") | |
| for kernel in kernels: | |
| print(kernel) | |
| if len(kernels) < 10: | |
| print( | |
| f"::warning::Prebuild may have failed: expected at least 10 kernel .so files, found {len(kernels)}. " | |
| "This can cause JIT compilation and OOM at runtime." | |
| ) | |
| else: | |
| print(f"Prebuild validation passed: {len(kernels)} kernels compiled") | |
| PY | |
| - name: Prepare Aiter wheel cache | |
| if: ${{ matrix.build_enabled }} | |
| run: | | |
| set -euo pipefail | |
| rm -rf aiter_wheels | |
| mkdir -p aiter_wheels | |
| cp dist/*.whl aiter_wheels/ | |
| ls -lh aiter_wheels | |
| - name: Save Aiter wheel cache | |
| if: ${{ matrix.build_enabled }} | |
| continue-on-error: true | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: aiter_wheels | |
| key: ${{ runner.os }}-aiter-wheel-${{ github.run_id }}-py${{ matrix.python_version }} | |
| - name: Upload wheel as artifact | |
| if: ${{ matrix.build_enabled }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.AITER_WHEEL_ARTIFACT_NAME }}-py${{ matrix.python_version }} | |
| path: dist/*.whl | |
| compression-level: 0 | |
| retention-days: 14 | |
| upload_s3_manifest: | |
| if: ${{ github.ref == 'refs/heads/main' && github.event_name != 'schedule' }} | |
| needs: [build_aiter_wheels] | |
| name: Upload wheels and manifest to S3 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Download all wheel artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: ${{ env.AITER_WHEEL_ARTIFACT_NAME }}-py* | |
| path: all-wheels | |
| merge-multiple: true | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: us-east-1 | |
| role-to-assume: arn:aws:iam::661452401056:role/framework-aiter-nightlies | |
| - name: Install AWS CLI | |
| run: | | |
| if ! command -v aws &> /dev/null; then | |
| curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | |
| unzip -q awscliv2.zip | |
| sudo ./aws/install | |
| rm -rf awscliv2.zip aws | |
| fi | |
| - name: Upload wheels and versioned manifest to S3 | |
| run: | | |
| set -euo pipefail | |
| S3_STAGING="s3://framework-whls-nightlies/whl-staging/gfx942-gfx950" | |
| S3_PUBLIC="https://rocm.frameworks-nightlies.amd.com/whl-staging/gfx942-gfx950" | |
| echo "=== Uploading all wheels to S3 ===" | |
| for WHL in all-wheels/amd_aiter*.whl; do | |
| WHL_NAME=$(basename "${WHL}") | |
| echo "Uploading ${WHL_NAME}..." | |
| aws s3 cp "${WHL}" "${S3_STAGING}/${WHL_NAME}" | |
| done | |
| echo "=== Generating versioned manifest ===" | |
| export MANIFEST_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") | |
| python3 - <<'PY' | |
| import json, glob, os, re, sys | |
| s3_public = os.environ["S3_PUBLIC"] | |
| timestamp = os.environ["MANIFEST_TIMESTAMP"] | |
| branch = os.environ["GITHUB_REF_NAME"] | |
| commit = os.environ["GITHUB_SHA"] | |
| wheels = sorted(glob.glob("all-wheels/amd_aiter*.whl")) | |
| if not wheels: | |
| print("ERROR: No amd_aiter wheels found", file=sys.stderr) | |
| sys.exit(1) | |
| wheels_by_tag = {} | |
| default_wheel_name = None | |
| default_wheel_url = None | |
| for whl_path in wheels: | |
| whl_name = os.path.basename(whl_path) | |
| whl_url = f"{s3_public}/{whl_name}" | |
| match = re.search(r"-(cp3\d+)-", whl_name) | |
| if match: | |
| tag = match.group(1) | |
| wheels_by_tag[tag] = {"wheel_name": whl_name, "wheel_url": whl_url} | |
| default_wheel_name = whl_name | |
| default_wheel_url = whl_url | |
| manifest = { | |
| "branch": branch, | |
| "timestamp": timestamp, | |
| "commit": commit, | |
| "wheels": wheels_by_tag, | |
| "wheel_name": default_wheel_name, | |
| "wheel_url": default_wheel_url, | |
| } | |
| with open("latest-main-wheel.json", "w") as f: | |
| json.dump(manifest, f, indent=2) | |
| f.write("\n") | |
| print(f"Manifest: {json.dumps(manifest, indent=2)}") | |
| PY | |
| aws s3 cp latest-main-wheel.json \ | |
| "${S3_STAGING}/main/latest.json" \ | |
| --content-type application/json | |
| echo "Uploaded versioned manifest" | |
| env: | |
| S3_PUBLIC: "https://rocm.frameworks-nightlies.amd.com/whl-staging/gfx942-gfx950" | |
| split_aiter_tests: | |
| if: ${{ !github.event.pull_request || github.event.pull_request.draft == false }} | |
| runs-on: ubuntu-latest | |
| needs: [check-signal, build_aiter_wheels] | |
| outputs: | |
| shard_count: 8 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - name: Split Aiter Tests (8 shards) | |
| run: ./.github/scripts/split_tests.sh --shards 8 --test-type aiter | |
| - name: Upload test shard lists as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: aiter_shards | |
| path: aiter_shard_*.list | |
| retention-days: 7 | |
| prepare_triton_wheel: | |
| if: ${{ !github.event.pull_request || github.event.pull_request.draft == false }} | |
| needs: check-signal | |
| uses: ./.github/workflows/prepare-triton-wheel.yaml | |
| with: | |
| docker-image: rocm/pytorch:latest | |
| artifact-name: triton_wheelhouse | |
| retention-days: 3 | |
| standard: | |
| if: >- | |
| (!github.event.pull_request || github.event.pull_request.draft == false) && | |
| github.event.action != 'labeled' | |
| name: Standard Tests (1 GPU) | |
| needs: [build_aiter_wheels, split_aiter_tests, prepare_triton_wheel] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 0 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 1 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 2 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 3 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 4 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 5 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 6 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 7 | |
| - runner: aiter-1gpu-runner | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 0 | |
| - runner: aiter-1gpu-runner | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 1 | |
| - runner: aiter-1gpu-runner | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 2 | |
| - runner: aiter-1gpu-runner | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 3 | |
| - runner: aiter-1gpu-runner | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 4 | |
| - runner: aiter-1gpu-runner | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 5 | |
| - runner: aiter-1gpu-runner | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 6 | |
| - runner: aiter-1gpu-runner | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 7 | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - name: Download test shard lists | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 15 | |
| with: | |
| name: aiter_shards | |
| - name: List test shard files | |
| run: | | |
| ls -l aiter_shard_*.list | |
| - name: Export test file list for this shard as env | |
| id: set_shard_files | |
| run: | | |
| echo "AITER_TEST=$(cat aiter_shard_${{ matrix.shard_idx }}.list)" >> $GITHUB_ENV | |
| echo "$AITER_TEST" | |
| - name: Restore Aiter wheel cache | |
| id: restore_aiter_wheel | |
| continue-on-error: true | |
| uses: actions/cache/restore@v4 | |
| env: | |
| SEGMENT_DOWNLOAD_TIMEOUT_MINS: 30 | |
| with: | |
| path: aiter_wheels | |
| key: ${{ runner.os }}-aiter-wheel-${{ github.run_id }}-py3.12 | |
| - name: Download Aiter wheel artifact | |
| if: steps.restore_aiter_wheel.outputs.cache-hit != 'true' | |
| id: download_aiter_wheel | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 30 | |
| continue-on-error: true | |
| with: | |
| name: ${{ env.AITER_WHEEL_ARTIFACT_NAME }}-py3.12 | |
| path: aiter_wheels | |
| - name: Retry Download Aiter wheel artifact | |
| if: steps.restore_aiter_wheel.outputs.cache-hit != 'true' && steps.download_aiter_wheel.outcome == 'failure' | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 30 | |
| with: | |
| name: ${{ env.AITER_WHEEL_ARTIFACT_NAME }}-py3.12 | |
| path: aiter_wheels | |
| - name: Download Triton wheel artifact | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 30 | |
| continue-on-error: true | |
| with: | |
| name: ${{ env.TRITON_WHEEL_ARTIFACT_NAME }} | |
| path: triton_wheels | |
| - name: Docker login | |
| if: ${{ !github.event.pull_request || !github.event.pull_request.head.repo.fork }} | |
| env: | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} | |
| run: | | |
| for attempt in 1 2 3; do | |
| if echo "$DOCKER_PASSWORD" | docker login -u rocmshared --password-stdin; then | |
| echo "Docker login succeeded on attempt ${attempt}" | |
| exit 0 | |
| fi | |
| echo "Docker login attempt ${attempt} failed" | |
| if [ "${attempt}" != 3 ]; then | |
| sleep 10 | |
| fi | |
| done | |
| echo "Docker login failed after 3 attempts, continuing anyway" | |
| exit 0 | |
| - name: Run the container | |
| run: | | |
| set -ex | |
| echo "Starting container: aiter_test" | |
| if [ -f "/etc/podinfo/gha-render-devices" ]; then | |
| DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices) | |
| else | |
| DEVICE_FLAG="--device /dev/dri" | |
| fi | |
| IMAGE_TAG=${{ env.DOCKER_IMAGE }} | |
| docker run -dt \ | |
| --device=/dev/kfd $DEVICE_FLAG \ | |
| --shm-size=16G \ | |
| --network=host \ | |
| --group-add $(getent group render | cut -d: -f3) \ | |
| --group-add $(getent group video | cut -d: -f3) \ | |
| -e AITER_TEST="${AITER_TEST}" \ | |
| -e TRITON_WHEEL_DIR=/workspace/triton_wheels \ | |
| -v "${{ github.workspace }}:/workspace" \ | |
| -w /workspace \ | |
| --name aiter_test \ | |
| $IMAGE_TAG | |
| - name: Check GPU visibility | |
| run: | | |
| set -euo pipefail | |
| if docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -lc './.github/scripts/aiter_gpu_visibility_check.sh'; then | |
| echo "GPU visibility check completed" | |
| else | |
| status=$? | |
| echo "::warning::GPU visibility check failed with exit code ${status}; continuing" | |
| fi | |
| - name: Sync CK submodule | |
| run: | | |
| set -ex | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| echo "Nightly build: syncing latest CK from develop branch..." | |
| git submodule set-branch --branch develop 3rdparty/composable_kernel | |
| git submodule sync | |
| git submodule update --init --recursive --remote --jobs 4 | |
| else | |
| echo "Using pinned CK commit..." | |
| git submodule sync | |
| git submodule update --init --recursive --depth 1 --jobs 4 | |
| fi | |
| - name: Install Aiter wheel | |
| run: | | |
| set -euo pipefail | |
| ls -lh aiter_wheels | |
| shopt -s nullglob | |
| wheels=(aiter_wheels/*.whl) | |
| if [ "${#wheels[@]}" -ne 1 ]; then | |
| echo "::error::Expected exactly one Aiter wheel artifact, found ${#wheels[@]}" | |
| exit 1 | |
| fi | |
| AITER_WHEEL_PATH="/workspace/${wheels[0]}" | |
| echo "Installing Aiter wheel: ${AITER_WHEEL_PATH}" | |
| docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -lc " | |
| pip uninstall -y amd-aiter aiter || true | |
| pip install -r requirements.txt | |
| pip install --upgrade pandas pyzmq einops numpy==1.26.2 | |
| pip install --upgrade 'pybind11>=3.0.1' | |
| pip install --upgrade 'ninja>=1.11.1' | |
| pip install tabulate | |
| pip install '${AITER_WHEEL_PATH}' | |
| pip show amd-aiter | |
| " | |
| - name: Sync prebuilt kernels into source tree | |
| run: | | |
| set -euo pipefail | |
| docker exec -i \ | |
| -w /workspace \ | |
| aiter_test \ | |
| python3 - <<'PY' | |
| from importlib.metadata import distribution | |
| from pathlib import Path | |
| from shutil import copy2 | |
| dist = distribution("amd-aiter") | |
| installed_jit = Path(dist.locate_file("aiter/jit")).resolve() | |
| workspace_jit = Path("/workspace/aiter/jit").resolve() | |
| kernels = sorted(installed_jit.glob("*.so")) | |
| if not kernels: | |
| raise SystemExit(f"No prebuilt kernels found in {installed_jit}") | |
| workspace_jit.mkdir(parents=True, exist_ok=True) | |
| for kernel in kernels: | |
| copy2(kernel, workspace_jit / kernel.name) | |
| print( | |
| f"Synced {len(kernels)} prebuilt kernels from {installed_jit} " | |
| f"to {workspace_jit}" | |
| ) | |
| PY | |
| - name: Install triton | |
| run: | | |
| # Ensure install_triton.sh is available even when the PR branch | |
| # was created before the script was added to main (#2959). | |
| # The workflow file always comes from the base branch, so we | |
| # materialize the script from the base ref when missing. | |
| if [ ! -f .github/scripts/install_triton.sh ]; then | |
| BASE_REF="${{ github.event.pull_request.base.ref || github.ref_name }}" | |
| mkdir -p .github/scripts | |
| curl -fsSL \ | |
| "https://raw.githubusercontent.com/${{ github.repository }}/${BASE_REF}/.github/scripts/install_triton.sh" \ | |
| -o .github/scripts/install_triton.sh | |
| chmod +x .github/scripts/install_triton.sh | |
| fi | |
| docker exec -w /workspace aiter_test \ | |
| bash -c "./.github/scripts/install_triton.sh && pip show triton" | |
| - name: Show Aiter version | |
| run: | | |
| set -ex | |
| docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -c "pip show amd-aiter || true" | |
| - name: Tests | |
| timeout-minutes: 90 | |
| run: | | |
| set -ex | |
| docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -c "SHARD_TOTAL=${{ matrix.shard_total }} SHARD_IDX=${{ matrix.shard_idx }} ./.github/scripts/aiter_test.sh" | |
| - name: Collect test logs | |
| if: always() | |
| run: | | |
| echo "Collecting test logs..." | |
| echo "Aiter Operator Tests Summary:" >> $GITHUB_STEP_SUMMARY | |
| python3 ./.github/scripts/collect_logs.py latest_test.log >> $GITHUB_STEP_SUMMARY | |
| - name: Upload test logs | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: standard-test-log-${{ matrix.runner }}-shard-${{ matrix.shard_idx }} | |
| path: | | |
| latest_test.log | |
| tuned_op_bench.csv | |
| if-no-files-found: warn | |
| retention-days: 7 | |
| - name: Cleanup container | |
| if: always() | |
| run: | | |
| docker rm -f aiter_test || true | |
| standard-test-finish: | |
| if: >- | |
| !github.event.pull_request.draft && | |
| github.event.action != 'labeled' | |
| name: Standard Test Results | |
| runs-on: ubuntu-latest | |
| needs: [standard] | |
| steps: | |
| - name: Download all test logs | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 15 | |
| with: | |
| pattern: standard-test-log-*-shard-* | |
| path: . | |
| - name: List test logs | |
| run: | | |
| ls -l standard-test-log-* | |
| - name: Check Standard Test Results | |
| run: | | |
| set -ex | |
| echo "Checking Standard Test Results..." | |
| all_passed=true | |
| for shard in {0..7}; do | |
| for runner in {linux-aiter-mi35x-1,aiter-1gpu-runner}; do | |
| if [ ! -f standard-test-log-${runner}-shard-${shard}/latest_test.log ]; then | |
| echo "Test report for ${runner} shard ${shard} not found." | |
| all_passed=false | |
| break | |
| fi | |
| done | |
| done | |
| if [ "$all_passed" = true ]; then | |
| echo "All tests passed." | |
| else | |
| echo "Test failures or errors detected." | |
| exit 1 | |
| fi | |
| multi-gpu: | |
| name: Multi-GPU Tests (8 GPU) | |
| if: github.ref == 'refs/heads/main' | |
| needs: [build_aiter_wheels, prepare_triton_wheel] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - runner: linux-aiter-mi35x-8 | |
| label: MI35X | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - name: Restore Aiter wheel cache | |
| id: restore_aiter_wheel | |
| continue-on-error: true | |
| uses: actions/cache/restore@v4 | |
| env: | |
| SEGMENT_DOWNLOAD_TIMEOUT_MINS: 30 | |
| with: | |
| path: aiter_wheels | |
| key: ${{ runner.os }}-aiter-wheel-${{ github.run_id }}-py3.12 | |
| - name: Download Aiter wheel artifact | |
| if: steps.restore_aiter_wheel.outputs.cache-hit != 'true' | |
| id: download_aiter_wheel | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 30 | |
| continue-on-error: true | |
| with: | |
| name: ${{ env.AITER_WHEEL_ARTIFACT_NAME }}-py3.12 | |
| path: aiter_wheels | |
| - name: Retry Download Aiter wheel artifact | |
| if: steps.restore_aiter_wheel.outputs.cache-hit != 'true' && steps.download_aiter_wheel.outcome == 'failure' | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 30 | |
| with: | |
| name: ${{ env.AITER_WHEEL_ARTIFACT_NAME }}-py3.12 | |
| path: aiter_wheels | |
| - name: Download Triton wheel artifact | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 30 | |
| continue-on-error: true | |
| with: | |
| name: ${{ env.TRITON_WHEEL_ARTIFACT_NAME }} | |
| path: triton_wheels | |
| - name: Docker login | |
| if: ${{ !github.event.pull_request || !github.event.pull_request.head.repo.fork }} | |
| env: | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} | |
| run: | | |
| for attempt in 1 2 3; do | |
| if echo "$DOCKER_PASSWORD" | docker login -u rocmshared --password-stdin; then | |
| echo "Docker login succeeded on attempt ${attempt}" | |
| exit 0 | |
| fi | |
| echo "Docker login attempt ${attempt} failed" | |
| if [ "${attempt}" != 3 ]; then | |
| sleep 10 | |
| fi | |
| done | |
| echo "Docker login failed after 3 attempts, continuing anyway" | |
| exit 0 | |
| - name: Run the container | |
| run: | | |
| set -ex | |
| echo "Starting container: aiter_test" | |
| if [ -f "/etc/podinfo/gha-render-devices" ]; then | |
| DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices) | |
| else | |
| DEVICE_FLAG="--device /dev/dri" | |
| fi | |
| IMAGE_TAG=${{ env.DOCKER_IMAGE }} | |
| docker run -dt \ | |
| --device=/dev/kfd $DEVICE_FLAG \ | |
| --shm-size=16G \ | |
| --network=host \ | |
| --group-add $(getent group render | cut -d: -f3) \ | |
| --group-add $(getent group video | cut -d: -f3) \ | |
| -e TRITON_WHEEL_DIR=/workspace/triton_wheels \ | |
| -v "${{ github.workspace }}:/workspace" \ | |
| -w /workspace \ | |
| --name aiter_test \ | |
| $IMAGE_TAG | |
| - name: Check GPU visibility | |
| run: | | |
| set -euo pipefail | |
| if docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -lc './.github/scripts/aiter_gpu_visibility_check.sh'; then | |
| echo "GPU visibility check completed" | |
| else | |
| status=$? | |
| echo "::warning::GPU visibility check failed with exit code ${status}; continuing" | |
| fi | |
| - name: Sync CK submodule | |
| run: | | |
| set -ex | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| echo "Nightly build: syncing latest CK from develop branch..." | |
| git submodule set-branch --branch develop 3rdparty/composable_kernel | |
| git submodule sync | |
| git submodule update --init --recursive --remote --jobs 4 | |
| else | |
| echo "Using pinned CK commit..." | |
| git submodule sync | |
| git submodule update --init --recursive --depth 1 --jobs 4 | |
| fi | |
| - name: Install Aiter wheel | |
| run: | | |
| set -euo pipefail | |
| ls -lh aiter_wheels | |
| shopt -s nullglob | |
| wheels=(aiter_wheels/*.whl) | |
| if [ "${#wheels[@]}" -ne 1 ]; then | |
| echo "::error::Expected exactly one Aiter wheel artifact, found ${#wheels[@]}" | |
| exit 1 | |
| fi | |
| AITER_WHEEL_PATH="/workspace/${wheels[0]}" | |
| echo "Installing Aiter wheel: ${AITER_WHEEL_PATH}" | |
| docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -lc " | |
| pip uninstall -y amd-aiter aiter || true | |
| pip install -r requirements.txt | |
| pip install --upgrade pandas pyzmq einops numpy==1.26.2 | |
| pip install --upgrade 'pybind11>=3.0.1' | |
| pip install --upgrade 'ninja>=1.11.1' | |
| pip install tabulate | |
| pip install '${AITER_WHEEL_PATH}' | |
| pip show amd-aiter | |
| " | |
| - name: Sync prebuilt kernels into source tree | |
| run: | | |
| set -euo pipefail | |
| docker exec -i \ | |
| -w /workspace \ | |
| aiter_test \ | |
| python3 - <<'PY' | |
| from importlib.metadata import distribution | |
| from pathlib import Path | |
| from shutil import copy2 | |
| dist = distribution("amd-aiter") | |
| installed_jit = Path(dist.locate_file("aiter/jit")).resolve() | |
| workspace_jit = Path("/workspace/aiter/jit").resolve() | |
| kernels = sorted(installed_jit.glob("*.so")) | |
| if not kernels: | |
| raise SystemExit(f"No prebuilt kernels found in {installed_jit}") | |
| workspace_jit.mkdir(parents=True, exist_ok=True) | |
| for kernel in kernels: | |
| copy2(kernel, workspace_jit / kernel.name) | |
| print( | |
| f"Synced {len(kernels)} prebuilt kernels from {installed_jit} " | |
| f"to {workspace_jit}" | |
| ) | |
| PY | |
| - name: Install triton | |
| run: | | |
| # Ensure install_triton.sh is available even when the PR branch | |
| # was created before the script was added to main (#2959). | |
| # The workflow file always comes from the base branch, so we | |
| # materialize the script from the base ref when missing. | |
| if [ ! -f .github/scripts/install_triton.sh ]; then | |
| BASE_REF="${{ github.event.pull_request.base.ref || github.ref_name }}" | |
| mkdir -p .github/scripts | |
| curl -fsSL \ | |
| "https://raw.githubusercontent.com/${{ github.repository }}/${BASE_REF}/.github/scripts/install_triton.sh" \ | |
| -o .github/scripts/install_triton.sh | |
| chmod +x .github/scripts/install_triton.sh | |
| fi | |
| docker exec -w /workspace aiter_test \ | |
| bash -c "./.github/scripts/install_triton.sh && pip show triton" | |
| - name: Show Aiter version | |
| run: | | |
| set -ex | |
| docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -c "pip show amd-aiter || true" | |
| - name: Tests | |
| timeout-minutes: 75 | |
| run: | | |
| set -ex | |
| docker exec \ | |
| -e MULTIGPU=TRUE \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -c "./.github/scripts/aiter_test.sh" | |
| - name: Upload test logs | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: multigpu-test-${{ matrix.runner }} | |
| path: latest_test.log | |
| retention-days: 7 | |
| - name: Cleanup container | |
| if: always() | |
| run: | | |
| docker rm -f aiter_test || true | |
| - name: Clean up Rocm processes | |
| if: always() | |
| run: | | |
| ./.github/scripts/clean_up_rocm.sh | |
| tuned_op_bench: | |
| # Tuned operator perf regression check. | |
| # - PR: pull last main baseline CSV, compare current shard's CSV, warn-only | |
| # - push to main / workflow_dispatch: publish current CSV as next baseline | |
| # Only consumes csv from linux-aiter-mi35x-1 runner to avoid cross-arch noise. | |
| name: Tuned Op Bench | |
| if: >- | |
| always() && | |
| !cancelled() && | |
| !github.event.pull_request.draft && | |
| github.event.action != 'labeled' && | |
| github.event_name != 'schedule' | |
| runs-on: ubuntu-latest | |
| needs: [standard] | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download standard test logs (mi35x only) | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 15 | |
| continue-on-error: true | |
| with: | |
| pattern: standard-test-log-linux-aiter-mi35x-1-shard-* | |
| path: /tmp/logs/ | |
| - name: Locate current tuned_op_bench.csv | |
| id: current | |
| run: | | |
| shopt -s nullglob | |
| csv_files=(/tmp/logs/*/tuned_op_bench.csv) | |
| if [[ ${#csv_files[@]} -eq 0 ]]; then | |
| echo "::warning::No tuned op benchmark CSV found in any mi35x shard; skipping tuned_op_bench" | |
| echo "found=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| echo "Merging tuned op benchmark CSVs:" | |
| printf ' %s\n' "${csv_files[@]}" | |
| python3 - "${csv_files[@]}" <<'PY' | |
| import csv | |
| import sys | |
| rows = [] | |
| fieldnames = [] | |
| seen = set() | |
| for path in sys.argv[1:]: | |
| with open(path, newline="") as f: | |
| reader = csv.DictReader(f) | |
| if not reader.fieldnames: | |
| continue | |
| for name in reader.fieldnames: | |
| if name not in seen: | |
| fieldnames.append(name) | |
| seen.add(name) | |
| rows.extend(reader) | |
| if "us" not in seen: | |
| raise SystemExit("merged tuned op benchmark CSV is missing required `us` column") | |
| with open("/tmp/current.csv", "w", newline="") as f: | |
| writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") | |
| writer.writeheader() | |
| for row in rows: | |
| writer.writerow({name: row.get(name, "") for name in fieldnames}) | |
| print(f"Merged {len(rows)} row(s) into /tmp/current.csv") | |
| PY | |
| echo "Current merged CSV: /tmp/current.csv ($(wc -l < /tmp/current.csv) lines)" | |
| echo "found=true" >> "$GITHUB_OUTPUT" | |
| # ── PR path: compare vs baseline ── | |
| - name: Fetch baseline from PR base.sha | |
| if: steps.current.outputs.found == 'true' && github.event_name == 'pull_request' | |
| id: baseline_pinned | |
| continue-on-error: true | |
| uses: dawidd6/action-download-artifact@v3 | |
| timeout-minutes: 15 | |
| with: | |
| workflow: aiter-test.yaml | |
| commit: ${{ github.event.pull_request.base.sha }} | |
| name: tuned-op-bench-${{ github.event.pull_request.base.sha }} | |
| path: /tmp/baseline_pinned/ | |
| if_no_artifact_found: warn | |
| - name: Fallback — fetch baseline from latest main | |
| if: >- | |
| steps.current.outputs.found == 'true' && | |
| github.event_name == 'pull_request' | |
| id: baseline_main | |
| continue-on-error: true | |
| uses: dawidd6/action-download-artifact@v3 | |
| timeout-minutes: 15 | |
| with: | |
| workflow: aiter-test.yaml | |
| branch: main | |
| name_is_regexp: true | |
| name: ^tuned-op-bench-[a-f0-9]+$ | |
| path: /tmp/baseline_main/ | |
| if_no_artifact_found: warn | |
| - name: Compare | |
| if: steps.current.outputs.found == 'true' && github.event_name == 'pull_request' | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| CURR_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| set -euo pipefail | |
| baseline_csv="" | |
| if [[ -f /tmp/baseline_pinned/tuned_op_bench.csv ]]; then | |
| baseline_csv=/tmp/baseline_pinned/tuned_op_bench.csv | |
| echo "Using baseline pinned to PR.base.sha=${BASE_SHA:0:7}" | |
| else | |
| # fallback: pick first match under /tmp/baseline_main/* | |
| shopt -s nullglob | |
| candidates=(/tmp/baseline_main/*/tuned_op_bench.csv /tmp/baseline_main/tuned_op_bench.csv) | |
| for c in "${candidates[@]}"; do | |
| if [[ -f "$c" ]]; then | |
| baseline_csv="$c" | |
| echo "Using fallback baseline from latest main: $c" | |
| break | |
| fi | |
| done | |
| fi | |
| if [[ -z "$baseline_csv" ]]; then | |
| echo "::warning::No tuned op benchmark baseline found (neither pinned PR.base.sha nor latest main); skipping compare." | |
| { | |
| echo "## Tuned Op Bench" | |
| echo | |
| echo "_No baseline available — first run on this branch or main hasn't published baseline yet._" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| echo "## Tuned Op Bench (vs baseline)" >> "$GITHUB_STEP_SUMMARY" | |
| echo '```' >> "$GITHUB_STEP_SUMMARY" | |
| bash .github/scripts/check_tuned_op_regression.sh \ | |
| "$baseline_csv" /tmp/current.csv \ | |
| | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo '```' >> "$GITHUB_STEP_SUMMARY" | |
| # ── main push / workflow_dispatch path: publish baseline ── | |
| - name: Stage baseline payload | |
| if: >- | |
| steps.current.outputs.found == 'true' && | |
| (github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| || github.event_name == 'workflow_dispatch') | |
| run: | | |
| mkdir -p /tmp/publish | |
| cp /tmp/current.csv /tmp/publish/tuned_op_bench.csv | |
| python3 -c " | |
| import json, os, datetime | |
| meta = { | |
| 'commit': os.environ['GITHUB_SHA'], | |
| 'ref': os.environ['GITHUB_REF'], | |
| 'event': os.environ['GITHUB_EVENT_NAME'], | |
| 'runner_pool': 'linux-aiter-mi35x-1', | |
| 'gpu_arch_list': os.environ.get('GPU_ARCH_LIST', ''), | |
| 'ran_at': datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'), | |
| } | |
| with open('/tmp/publish/metadata.json', 'w') as f: | |
| json.dump(meta, f, indent=2) | |
| print(json.dumps(meta, indent=2)) | |
| " | |
| ls -la /tmp/publish/ | |
| wc -l /tmp/publish/tuned_op_bench.csv | |
| - name: Publish baseline artifact | |
| if: >- | |
| steps.current.outputs.found == 'true' && | |
| (github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| || github.event_name == 'workflow_dispatch') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: tuned-op-bench-${{ github.sha }} | |
| path: /tmp/publish/ | |
| retention-days: 90 |