Aiter Test #214
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Aiter Test | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| branches: [main] # Triggers on PRs targeting `main` | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 22 * * *' # 6:00 AM Beijing Time (UTC+8) | |
| concurrency: | |
| # Keep scheduled main runs from blocking push-triggered validation. | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| env: | |
| GPU_ARCH_LIST: "gfx942;gfx950" | |
| AITER_TEST: "op_tests" | |
| AITER_WHEEL_ARTIFACT_NAME: aiter-whl-${{ github.run_id }} | |
| AITER_PREBUILD_MAX_JOBS: "64" | |
| jobs: | |
| detect_aiter_test_scope: | |
| name: Detect Aiter Test Scope | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| run_tests: ${{ steps.detect.outputs.run_tests }} | |
| reason: ${{ steps.detect.outputs.reason }} | |
| steps: | |
| - name: Detect whether Aiter tests are required | |
| id: detect | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| PR_DRAFT: ${{ github.event.pull_request.draft }} | |
| run: | | |
| set -euo pipefail | |
| if [ "${GITHUB_EVENT_NAME}" != "pull_request" ]; then | |
| echo "run_tests=true" >> "$GITHUB_OUTPUT" | |
| echo "reason=non_pull_request_event" >> "$GITHUB_OUTPUT" | |
| echo "Aiter tests are required for ${GITHUB_EVENT_NAME}." >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| if [ "${PR_DRAFT}" = "true" ]; then | |
| echo "run_tests=false" >> "$GITHUB_OUTPUT" | |
| echo "reason=draft_pull_request" >> "$GITHUB_OUTPUT" | |
| echo "Aiter tests are skipped because this pull request is a draft." >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| mapfile -t changed_files < <( | |
| gh api --paginate \ | |
| "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ | |
| --jq '.[].filename' | |
| ) | |
| run_tests=false | |
| reason=only_ignored_changed_files | |
| for file in "${changed_files[@]}"; do | |
| case "${file}" in | |
| .github/workflows/aiter-test.yaml) | |
| run_tests=true | |
| reason=matching_changed_files | |
| break | |
| ;; | |
| *.md|docs/*|LICENSE|.gitignore|.github/scripts/sglang_downstream.py|.github/workflows/*) | |
| ;; | |
| *) | |
| run_tests=true | |
| reason=matching_changed_files | |
| break | |
| ;; | |
| esac | |
| done | |
| echo "run_tests=${run_tests}" >> "$GITHUB_OUTPUT" | |
| echo "reason=${reason}" >> "$GITHUB_OUTPUT" | |
| { | |
| echo "## Aiter Test Scope" | |
| echo | |
| echo "- Changed files: ${#changed_files[@]}" | |
| echo "- Aiter tests required: ${run_tests}" | |
| echo "- Reason: ${reason}" | |
| if [ "${run_tests}" = "true" ]; then | |
| echo "Aiter tests are required because at least one changed file matches the Aiter test scope." | |
| else | |
| echo "Aiter tests are not required for this pull request." | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| ci_config: | |
| uses: ./.github/workflows/ci-config.yaml | |
| check-signal: | |
| if: ${{ needs.detect_aiter_test_scope.outputs.run_tests == 'true' }} | |
| needs: detect_aiter_test_scope | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Wait for Checks workflow | |
| run: ./.github/scripts/check_signal.sh | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_SHA: ${{ github.sha }} | |
| build_aiter_wheels: | |
| if: ${{ needs.detect_aiter_test_scope.outputs.run_tests == 'true' }} | |
| runs-on: build-only-aiter | |
| needs: [detect_aiter_test_scope, check-signal, ci_config] | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - python_version: "3.10" | |
| docker_image: ${{ needs.ci_config.outputs.pytorch_py310_image }} | |
| build_enabled: ${{ github.ref == 'refs/heads/main' }} | |
| - python_version: "3.12" | |
| docker_image: ${{ needs.ci_config.outputs.pytorch_py312_image }} | |
| build_enabled: true | |
| steps: | |
| # ---- Common steps (fork and non-fork) ---- | |
| - name: Checkout code | |
| if: ${{ matrix.build_enabled }} | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - name: Sync submodules for wheel build | |
| if: ${{ matrix.build_enabled }} | |
| run: | | |
| set -ex | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| echo "Nightly build: syncing latest CK from develop branch..." | |
| git submodule set-branch --branch develop 3rdparty/composable_kernel | |
| git submodule sync | |
| git submodule update --init --recursive --remote --jobs 4 | |
| else | |
| echo "Using pinned CK commit..." | |
| git submodule sync | |
| git submodule update --init --recursive --depth 1 --jobs 4 | |
| fi | |
| echo "CK commit: $(git -C 3rdparty/composable_kernel rev-parse HEAD)" | |
| - name: Docker login | |
| if: ${{ matrix.build_enabled && (!github.event.pull_request || !github.event.pull_request.head.repo.fork) }} | |
| env: | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} | |
| run: | | |
| for attempt in 1 2 3; do | |
| if echo "$DOCKER_PASSWORD" | docker login -u rocmshared --password-stdin; then | |
| echo "Docker login succeeded on attempt ${attempt}" | |
| exit 0 | |
| fi | |
| echo "Docker login attempt ${attempt} failed" | |
| if [ "${attempt}" != 3 ]; then | |
| sleep 10 | |
| fi | |
| done | |
| echo "Docker login failed after 3 attempts, continuing anyway" | |
| exit 0 | |
| - name: Build Aiter wheel in base container | |
| if: ${{ matrix.build_enabled }} | |
| run: | | |
| set -euo pipefail | |
| docker run --rm \ | |
| --network=host \ | |
| -e AITER_PREBUILD_MAX_JOBS="${{ env.AITER_PREBUILD_MAX_JOBS }}" \ | |
| -e AITER_RUNNER_NAME="${RUNNER_NAME:-unknown}" \ | |
| -v "${{ github.workspace }}:/workspace" \ | |
| -w /workspace \ | |
| ${{ matrix.docker_image }} \ | |
| bash -lc ' | |
| set -euo pipefail | |
| git config --global --add safe.directory /workspace && | |
| git -C /workspace rev-parse HEAD >/dev/null && | |
| shopt -s nullglob && | |
| rm -rf dist build aiter_meta ./*.egg-info && | |
| pip install -r requirements.txt && | |
| pip install --upgrade pandas pyzmq einops numpy==1.26.2 && | |
| pip install --upgrade "pybind11>=3.0.1" && | |
| pip install --upgrade "ninja>=1.11.1" && | |
| pip install --upgrade setuptools_scm && | |
| pip install tabulate && | |
| echo "Prebuilding kernels with GPU_ARCHS: ${{ env.GPU_ARCH_LIST }}, PREBUILD_KERNELS: 1, and MAX_JOBS: ${AITER_PREBUILD_MAX_JOBS}" && | |
| export PREBUILD_KERNELS=1 && | |
| export MAX_JOBS="${AITER_PREBUILD_MAX_JOBS}" && | |
| export GPU_ARCHS="${{ env.GPU_ARCH_LIST }}" && | |
| prebuild_start=$(date +%s) && | |
| set +e && | |
| python setup.py bdist_wheel 2>&1 | tee .aiter-prebuild.log | |
| build_status=${PIPESTATUS[0]} | |
| set -e | |
| prebuild_end=$(date +%s) | |
| { | |
| echo "BUILD_STATUS=${build_status}" | |
| echo "PREBUILD_START=${prebuild_start}" | |
| echo "PREBUILD_END=${prebuild_end}" | |
| } > .aiter-prebuild.env | |
| if [ "${build_status}" -ne 0 ]; then | |
| exit "${build_status}" | |
| fi | |
| ls -lh dist/*.whl | |
| ' | |
| - name: Summarize Aiter prebuild | |
| if: ${{ always() && matrix.build_enabled }} | |
| run: | | |
| set -euo pipefail | |
| if [ ! -f .aiter-prebuild.env ]; then | |
| echo "::warning::Aiter prebuild metadata was not generated" | |
| exit 0 | |
| fi | |
| . ./.aiter-prebuild.env | |
| AITER_RUNNER_NAME="${RUNNER_NAME:-unknown}" \ | |
| GPU_ARCHS="${{ env.GPU_ARCH_LIST }}" \ | |
| PREBUILD_KERNELS=1 \ | |
| MAX_JOBS="${{ env.AITER_PREBUILD_MAX_JOBS }}" \ | |
| python3 .github/scripts/aiter_prebuild_summary.py \ | |
| --log .aiter-prebuild.log \ | |
| --build-status "${BUILD_STATUS}" \ | |
| --start "${PREBUILD_START}" \ | |
| --end "${PREBUILD_END}" | |
| - name: Verify prebuilt kernels in wheel | |
| if: ${{ matrix.build_enabled }} | |
| run: | | |
| set -euo pipefail | |
| python3 - <<'PY' | |
| import glob | |
| import zipfile | |
| wheels = glob.glob("dist/*.whl") | |
| if len(wheels) != 1: | |
| raise SystemExit(f"Expected exactly one wheel in dist/, found {len(wheels)}") | |
| wheel = wheels[0] | |
| with zipfile.ZipFile(wheel) as zf: | |
| kernels = sorted( | |
| name for name in zf.namelist() | |
| if name.startswith("aiter/jit/") and name.endswith(".so") | |
| ) | |
| print("=== Prebuilt kernel validation ===") | |
| print(f"Wheel: {wheel}") | |
| print(f"Prebuilt kernel .so files: {len(kernels)}") | |
| for kernel in kernels: | |
| print(kernel) | |
| if len(kernels) < 10: | |
| print( | |
| f"::warning::Prebuild may have failed: expected at least 10 kernel .so files, found {len(kernels)}. " | |
| "This can cause JIT compilation and OOM at runtime." | |
| ) | |
| else: | |
| print(f"Prebuild validation passed: {len(kernels)} kernels compiled") | |
| PY | |
| - name: Prepare Aiter wheel cache | |
| if: ${{ matrix.build_enabled }} | |
| run: | | |
| set -euo pipefail | |
| rm -rf aiter_wheels | |
| mkdir -p aiter_wheels | |
| cp dist/*.whl aiter_wheels/ | |
| ls -lh aiter_wheels | |
| - name: Save Aiter wheel cache | |
| if: ${{ matrix.build_enabled }} | |
| continue-on-error: true | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: aiter_wheels | |
| key: ${{ runner.os }}-aiter-wheel-${{ github.run_id }}-py${{ matrix.python_version }} | |
| - name: Upload wheel as artifact | |
| if: ${{ matrix.build_enabled }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.AITER_WHEEL_ARTIFACT_NAME }}-py${{ matrix.python_version }} | |
| path: dist/*.whl | |
| compression-level: 0 | |
| retention-days: 14 | |
| upload_s3_manifest: | |
| if: ${{ github.ref == 'refs/heads/main' && github.event_name != 'schedule' }} | |
| needs: [build_aiter_wheels] | |
| name: Upload wheels and manifest to S3 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Download all wheel artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: ${{ env.AITER_WHEEL_ARTIFACT_NAME }}-py* | |
| path: all-wheels | |
| merge-multiple: true | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: us-east-1 | |
| role-to-assume: arn:aws:iam::661452401056:role/framework-aiter-nightlies | |
| - name: Install AWS CLI | |
| run: | | |
| if ! command -v aws &> /dev/null; then | |
| curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | |
| unzip -q awscliv2.zip | |
| sudo ./aws/install | |
| rm -rf awscliv2.zip aws | |
| fi | |
| - name: Upload wheels and versioned manifest to S3 | |
| run: | | |
| set -euo pipefail | |
| S3_STAGING="s3://framework-whls-nightlies/whl-staging/gfx942-gfx950" | |
| S3_PUBLIC="https://rocm.frameworks-nightlies.amd.com/whl-staging/gfx942-gfx950" | |
| echo "=== Uploading all wheels to S3 ===" | |
| for WHL in all-wheels/amd_aiter*.whl; do | |
| WHL_NAME=$(basename "${WHL}") | |
| echo "Uploading ${WHL_NAME}..." | |
| aws s3 cp "${WHL}" "${S3_STAGING}/${WHL_NAME}" \ | |
| --cache-control "public, max-age=31536000, immutable" | |
| done | |
| echo "=== Generating versioned manifest ===" | |
| export MANIFEST_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") | |
| python3 - <<'PY' | |
| import json, glob, os, re, sys | |
| from urllib.parse import quote | |
| s3_public = os.environ["S3_PUBLIC"] | |
| timestamp = os.environ["MANIFEST_TIMESTAMP"] | |
| branch = os.environ["GITHUB_REF_NAME"] | |
| commit = os.environ["GITHUB_SHA"] | |
| wheels = sorted(glob.glob("all-wheels/amd_aiter*.whl")) | |
| if not wheels: | |
| print("ERROR: No amd_aiter wheels found", file=sys.stderr) | |
| sys.exit(1) | |
| wheels_by_tag = {} | |
| default_wheel_name = None | |
| default_wheel_url = None | |
| for whl_path in wheels: | |
| whl_name = os.path.basename(whl_path) | |
| whl_url = f"{s3_public}/{quote(whl_name, safe='')}" | |
| match = re.search(r"-(cp3\d+)-", whl_name) | |
| if match: | |
| tag = match.group(1) | |
| wheels_by_tag[tag] = {"wheel_name": whl_name, "wheel_url": whl_url} | |
| default_wheel_name = whl_name | |
| default_wheel_url = whl_url | |
| manifest = { | |
| "branch": branch, | |
| "timestamp": timestamp, | |
| "commit": commit, | |
| "wheels": wheels_by_tag, | |
| "wheel_name": default_wheel_name, | |
| "wheel_url": default_wheel_url, | |
| } | |
| with open("latest-main-wheel.json", "w") as f: | |
| json.dump(manifest, f, indent=2) | |
| f.write("\n") | |
| print(f"Manifest: {json.dumps(manifest, indent=2)}") | |
| PY | |
| aws s3 cp latest-main-wheel.json \ | |
| "${S3_STAGING}/commits/${GITHUB_SHA}/latest.json" \ | |
| --content-type application/json \ | |
| --cache-control "public, max-age=31536000, immutable" | |
| aws s3 cp latest-main-wheel.json \ | |
| "${S3_STAGING}/main/latest.json" \ | |
| --content-type application/json \ | |
| --cache-control "no-store, max-age=0, must-revalidate" | |
| if [ -n "${CLOUDFRONT_DISTRIBUTION_ID:-}" ]; then | |
| aws cloudfront create-invalidation \ | |
| --distribution-id "${CLOUDFRONT_DISTRIBUTION_ID}" \ | |
| --paths "/whl-staging/gfx942-gfx950/main/latest.json" | |
| else | |
| echo "::warning::CLOUDFRONT_DISTRIBUTION_ID is not set; skipping latest.json invalidation" | |
| fi | |
| echo "Uploaded versioned manifest" | |
| env: | |
| S3_PUBLIC: "https://rocm.frameworks-nightlies.amd.com/whl-staging/gfx942-gfx950" | |
| CLOUDFRONT_DISTRIBUTION_ID: ${{ vars.AITER_WHEEL_CLOUDFRONT_DISTRIBUTION_ID }} | |
| split_aiter_tests: | |
| if: ${{ needs.detect_aiter_test_scope.outputs.run_tests == 'true' }} | |
| runs-on: ubuntu-latest | |
| needs: [detect_aiter_test_scope, check-signal, build_aiter_wheels] | |
| outputs: | |
| shard_count: 8 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - name: Split Aiter Tests (8 shards) | |
| run: ./.github/scripts/split_tests.sh --shards 8 --test-type aiter | |
| - name: Upload test shard lists as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: aiter_shards | |
| path: aiter_shard_*.list | |
| retention-days: 7 | |
| prepare_triton_wheel: | |
| if: ${{ needs.detect_aiter_test_scope.outputs.run_tests == 'true' }} | |
| needs: [detect_aiter_test_scope, check-signal, ci_config] | |
| uses: ./.github/workflows/prepare-triton-wheel.yaml | |
| with: | |
| docker-image: ${{ needs.ci_config.outputs.pytorch_py312_image }} | |
| artifact-name: ${{ needs.ci_config.outputs.triton_wheel_artifact_name }} | |
| retention-days: 3 | |
| standard: | |
| if: >- | |
| needs.detect_aiter_test_scope.outputs.run_tests == 'true' && | |
| github.event.action != 'labeled' | |
| name: Standard Tests (1 GPU) | |
| needs: [detect_aiter_test_scope, build_aiter_wheels, split_aiter_tests, prepare_triton_wheel, ci_config] | |
| env: | |
| DOCKER_IMAGE: ${{ needs.ci_config.outputs.pytorch_py312_image }} | |
| TRITON_WHEEL_ARTIFACT_NAME: ${{ needs.ci_config.outputs.triton_wheel_artifact_name }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 0 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 1 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 2 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 3 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 4 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 5 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 6 | |
| - runner: linux-aiter-mi35x-1 | |
| label: MI35X | |
| shard_total: 8 | |
| shard_idx: 7 | |
| - runner: linux-aiter-oci-mi300x-1 | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 0 | |
| - runner: linux-aiter-oci-mi300x-1 | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 1 | |
| - runner: linux-aiter-oci-mi300x-1 | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 2 | |
| - runner: linux-aiter-oci-mi300x-1 | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 3 | |
| - runner: linux-aiter-oci-mi300x-1 | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 4 | |
| - runner: linux-aiter-oci-mi300x-1 | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 5 | |
| - runner: linux-aiter-oci-mi300x-1 | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 6 | |
| - runner: linux-aiter-oci-mi300x-1 | |
| label: MI300X | |
| shard_total: 8 | |
| shard_idx: 7 | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - name: Network preflight before Aiter test artifact downloads | |
| if: ${{ startsWith(runner.name, 'linux-aiter-mi35x-1-') }} | |
| continue-on-error: true | |
| timeout-minutes: 5 | |
| run: | | |
| set +e | |
| echo "=== Runner info ===" | |
| echo "RUNNER_NAME=${RUNNER_NAME}" | |
| echo "RUNNER_OS=${RUNNER_OS}" | |
| echo "hostname=$(hostname -f 2>/dev/null || hostname)" | |
| date -u | |
| echo "=== DNS check ===" | |
| getent hosts github.com || true | |
| getent hosts api.github.com || true | |
| getent hosts productionresultssa4.blob.core.windows.net || true | |
| getent hosts productionresultssa8.blob.core.windows.net || true | |
| echo "=== GitHub and Azure Blob connectivity check ===" | |
| for url in \ | |
| https://api.github.com/rate_limit \ | |
| https://github.com/ROCm/aiter \ | |
| https://productionresultssa4.blob.core.windows.net/ \ | |
| https://productionresultssa8.blob.core.windows.net/ | |
| do | |
| curl -L -o /dev/null -sS \ | |
| --connect-timeout 10 \ | |
| --max-time 30 \ | |
| -w "url=${url} code=%{http_code} dns=%{time_namelookup}s connect=%{time_connect}s tls=%{time_appconnect}s ttfb=%{time_starttransfer}s total=%{time_total}s speed=%{speed_download}B/s\n" \ | |
| "$url" || true | |
| done | |
| - name: Download test shard lists | |
| id: download_aiter_shards | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 15 | |
| with: | |
| name: aiter_shards | |
| - name: List test shard files | |
| run: | | |
| ls -l aiter_shard_*.list | |
| - name: Export test file list for this shard as env | |
| id: set_shard_files | |
| run: | | |
| echo "AITER_TEST=$(cat aiter_shard_${{ matrix.shard_idx }}.list)" >> $GITHUB_ENV | |
| echo "$AITER_TEST" | |
| - name: Restore Aiter wheel cache | |
| id: restore_aiter_wheel | |
| continue-on-error: true | |
| uses: actions/cache/restore@v4 | |
| env: | |
| SEGMENT_DOWNLOAD_TIMEOUT_MINS: 30 | |
| with: | |
| path: aiter_wheels | |
| key: ${{ runner.os }}-aiter-wheel-${{ github.run_id }}-py3.12 | |
| - name: Download Aiter wheel artifact | |
| if: steps.restore_aiter_wheel.outputs.cache-hit != 'true' | |
| id: download_aiter_wheel | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 30 | |
| continue-on-error: true | |
| with: | |
| name: ${{ env.AITER_WHEEL_ARTIFACT_NAME }}-py3.12 | |
| path: aiter_wheels | |
| - name: Retry Download Aiter wheel artifact | |
| if: steps.restore_aiter_wheel.outputs.cache-hit != 'true' && steps.download_aiter_wheel.outcome == 'failure' | |
| id: retry_aiter_wheel | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 30 | |
| with: | |
| name: ${{ env.AITER_WHEEL_ARTIFACT_NAME }}-py3.12 | |
| path: aiter_wheels | |
| - name: Download Triton wheel artifact | |
| id: download_triton_wheel | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 30 | |
| continue-on-error: true | |
| with: | |
| name: ${{ env.TRITON_WHEEL_ARTIFACT_NAME }} | |
| path: triton_wheels | |
| - name: Speedtest after Aiter test artifact download failure | |
| if: ${{ always() && startsWith(runner.name, 'linux-aiter-mi35x-1-') && (steps.download_aiter_shards.outcome == 'failure' || steps.restore_aiter_wheel.outcome == 'failure' || steps.retry_aiter_wheel.outcome == 'failure' || steps.download_triton_wheel.outcome == 'failure') }} | |
| continue-on-error: true | |
| timeout-minutes: 5 | |
| run: | | |
| set +e | |
| echo "=== Runner info ===" | |
| echo "RUNNER_NAME=${RUNNER_NAME}" | |
| echo "RUNNER_OS=${RUNNER_OS}" | |
| echo "hostname=$(hostname -f 2>/dev/null || hostname)" | |
| date -u | |
| echo "=== speedtest-cli ===" | |
| python3 -m pip install --user --disable-pip-version-check speedtest-cli || true | |
| speedtest_bin="$(python3 -m site --user-base)/bin/speedtest-cli" | |
| timeout 120 "$speedtest_bin" --simple --secure || true | |
| - name: Docker login | |
| if: ${{ !github.event.pull_request || !github.event.pull_request.head.repo.fork }} | |
| env: | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} | |
| run: | | |
| for attempt in 1 2 3; do | |
| if echo "$DOCKER_PASSWORD" | docker login -u rocmshared --password-stdin; then | |
| echo "Docker login succeeded on attempt ${attempt}" | |
| exit 0 | |
| fi | |
| echo "Docker login attempt ${attempt} failed" | |
| if [ "${attempt}" != 3 ]; then | |
| sleep 10 | |
| fi | |
| done | |
| echo "Docker login failed after 3 attempts, continuing anyway" | |
| exit 0 | |
| - name: Run the container | |
| run: | | |
| set -ex | |
| echo "Starting container: aiter_test" | |
| if [ -f "/etc/podinfo/gha-render-devices" ]; then | |
| DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices) | |
| else | |
| DEVICE_FLAG="--device /dev/dri" | |
| fi | |
| IMAGE_TAG=${{ env.DOCKER_IMAGE }} | |
| docker run -dt \ | |
| --device=/dev/kfd $DEVICE_FLAG \ | |
| --shm-size=16G \ | |
| --network=host \ | |
| --group-add $(getent group render | cut -d: -f3) \ | |
| --group-add $(getent group video | cut -d: -f3) \ | |
| -e AITER_TEST="${AITER_TEST}" \ | |
| -e TRITON_WHEEL_DIR=/workspace/triton_wheels \ | |
| -v "${{ github.workspace }}:/workspace" \ | |
| -w /workspace \ | |
| --name aiter_test \ | |
| $IMAGE_TAG | |
| - name: Check GPU visibility | |
| run: | | |
| set -euo pipefail | |
| if docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -lc './.github/scripts/aiter_gpu_visibility_check.sh'; then | |
| echo "GPU visibility check completed" | |
| else | |
| status=$? | |
| echo "::warning::GPU visibility check failed with exit code ${status}; continuing" | |
| fi | |
| - name: Sync CK submodule | |
| run: | | |
| set -ex | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| echo "Nightly build: syncing latest CK from develop branch..." | |
| git submodule set-branch --branch develop 3rdparty/composable_kernel | |
| git submodule sync | |
| git submodule update --init --recursive --remote --jobs 4 | |
| else | |
| echo "Using pinned CK commit..." | |
| git submodule sync | |
| git submodule update --init --recursive --depth 1 --jobs 4 | |
| fi | |
| - name: Install Aiter wheel | |
| run: | | |
| set -euo pipefail | |
| ls -lh aiter_wheels | |
| shopt -s nullglob | |
| wheels=(aiter_wheels/*.whl) | |
| if [ "${#wheels[@]}" -ne 1 ]; then | |
| echo "::error::Expected exactly one Aiter wheel artifact, found ${#wheels[@]}" | |
| exit 1 | |
| fi | |
| AITER_WHEEL_PATH="/workspace/${wheels[0]}" | |
| echo "Installing Aiter wheel: ${AITER_WHEEL_PATH}" | |
| docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -lc " | |
| pip uninstall -y amd-aiter aiter || true | |
| pip install -r requirements.txt | |
| pip install --upgrade pandas pyzmq einops numpy==1.26.2 | |
| pip install --upgrade 'pybind11>=3.0.1' | |
| pip install --upgrade 'ninja>=1.11.1' | |
| pip install tabulate | |
| pip install '${AITER_WHEEL_PATH}' | |
| pip show amd-aiter | |
| " | |
| - name: Sync prebuilt kernels into source tree | |
| run: | | |
| set -euo pipefail | |
| docker exec -i \ | |
| -w /workspace \ | |
| aiter_test \ | |
| python3 - <<'PY' | |
| from importlib.metadata import distribution | |
| from pathlib import Path | |
| from shutil import copy2 | |
| dist = distribution("amd-aiter") | |
| installed_jit = Path(dist.locate_file("aiter/jit")).resolve() | |
| workspace_jit = Path("/workspace/aiter/jit").resolve() | |
| kernels = sorted(installed_jit.glob("*.so")) | |
| if not kernels: | |
| raise SystemExit(f"No prebuilt kernels found in {installed_jit}") | |
| workspace_jit.mkdir(parents=True, exist_ok=True) | |
| for kernel in kernels: | |
| copy2(kernel, workspace_jit / kernel.name) | |
| print( | |
| f"Synced {len(kernels)} prebuilt kernels from {installed_jit} " | |
| f"to {workspace_jit}" | |
| ) | |
| PY | |
| - name: Install triton | |
| run: | | |
| # Ensure install_triton.sh is available even when the PR branch | |
| # was created before the script was added to main (#2959). | |
| # The workflow file always comes from the base branch, so we | |
| # materialize the script from the base ref when missing. | |
| if [ ! -f .github/scripts/install_triton.sh ]; then | |
| BASE_REF="${{ github.event.pull_request.base.ref || github.ref_name }}" | |
| mkdir -p .github/scripts | |
| curl -fsSL \ | |
| "https://raw.githubusercontent.com/${{ github.repository }}/${BASE_REF}/.github/scripts/install_triton.sh" \ | |
| -o .github/scripts/install_triton.sh | |
| chmod +x .github/scripts/install_triton.sh | |
| fi | |
| docker exec -w /workspace aiter_test \ | |
| bash -c "./.github/scripts/install_triton.sh && pip show triton" | |
| - name: Show Aiter version | |
| run: | | |
| set -ex | |
| docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -c "pip show amd-aiter || true" | |
| - name: Tests | |
| timeout-minutes: 90 | |
| run: | | |
| set -ex | |
| docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -c "SHARD_TOTAL=${{ matrix.shard_total }} SHARD_IDX=${{ matrix.shard_idx }} ./.github/scripts/aiter_test.sh" | |
| - name: Collect test logs | |
| if: always() | |
| run: | | |
| echo "Collecting test logs..." | |
| echo "Aiter Operator Tests Summary:" >> $GITHUB_STEP_SUMMARY | |
| python3 ./.github/scripts/collect_logs.py latest_test.log >> $GITHUB_STEP_SUMMARY | |
| - name: Upload test logs | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: standard-test-log-${{ matrix.runner }}-shard-${{ matrix.shard_idx }} | |
| path: | | |
| latest_test.log | |
| tuned_op_bench.csv | |
| if-no-files-found: warn | |
| retention-days: 7 | |
| - name: Cleanup container | |
| if: always() | |
| run: | | |
| docker rm -f aiter_test || true | |
| standard-test-finish: | |
| if: >- | |
| needs.detect_aiter_test_scope.outputs.run_tests == 'true' && | |
| github.event.action != 'labeled' | |
| name: Standard Test Results | |
| runs-on: ubuntu-latest | |
| needs: [detect_aiter_test_scope, standard] | |
| steps: | |
| - name: Download all test logs | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 15 | |
| with: | |
| pattern: standard-test-log-*-shard-* | |
| path: . | |
| - name: List test logs | |
| run: | | |
| ls -l standard-test-log-* | |
| - name: Check Standard Test Results | |
| run: | | |
| set -ex | |
| echo "Checking Standard Test Results..." | |
| all_passed=true | |
| for shard in {0..7}; do | |
| for runner in {linux-aiter-mi35x-1,linux-aiter-oci-mi300x-1}; do | |
| if [ ! -f standard-test-log-${runner}-shard-${shard}/latest_test.log ]; then | |
| echo "Test report for ${runner} shard ${shard} not found." | |
| all_passed=false | |
| break | |
| fi | |
| done | |
| done | |
| if [ "$all_passed" = true ]; then | |
| echo "All tests passed." | |
| else | |
| echo "Test failures or errors detected." | |
| exit 1 | |
| fi | |
| multi-gpu: | |
| name: Multi-GPU Tests (8 GPU) | |
| if: github.ref == 'refs/heads/main' | |
| needs: [build_aiter_wheels, prepare_triton_wheel, ci_config] | |
| env: | |
| DOCKER_IMAGE: ${{ needs.ci_config.outputs.pytorch_py312_image }} | |
| TRITON_WHEEL_ARTIFACT_NAME: ${{ needs.ci_config.outputs.triton_wheel_artifact_name }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - runner: linux-aiter-do-mi350x-8 | |
| label: MI350X | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - name: Restore Aiter wheel cache | |
| id: restore_aiter_wheel | |
| continue-on-error: true | |
| uses: actions/cache/restore@v4 | |
| env: | |
| SEGMENT_DOWNLOAD_TIMEOUT_MINS: 30 | |
| with: | |
| path: aiter_wheels | |
| key: ${{ runner.os }}-aiter-wheel-${{ github.run_id }}-py3.12 | |
| - name: Download Aiter wheel artifact | |
| if: steps.restore_aiter_wheel.outputs.cache-hit != 'true' | |
| id: download_aiter_wheel | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 30 | |
| continue-on-error: true | |
| with: | |
| name: ${{ env.AITER_WHEEL_ARTIFACT_NAME }}-py3.12 | |
| path: aiter_wheels | |
| - name: Retry Download Aiter wheel artifact | |
| if: steps.restore_aiter_wheel.outputs.cache-hit != 'true' && steps.download_aiter_wheel.outcome == 'failure' | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 30 | |
| with: | |
| name: ${{ env.AITER_WHEEL_ARTIFACT_NAME }}-py3.12 | |
| path: aiter_wheels | |
| - name: Download Triton wheel artifact | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 30 | |
| continue-on-error: true | |
| with: | |
| name: ${{ env.TRITON_WHEEL_ARTIFACT_NAME }} | |
| path: triton_wheels | |
| - name: Docker login | |
| if: ${{ !github.event.pull_request || !github.event.pull_request.head.repo.fork }} | |
| env: | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} | |
| run: | | |
| for attempt in 1 2 3; do | |
| if echo "$DOCKER_PASSWORD" | docker login -u rocmshared --password-stdin; then | |
| echo "Docker login succeeded on attempt ${attempt}" | |
| exit 0 | |
| fi | |
| echo "Docker login attempt ${attempt} failed" | |
| if [ "${attempt}" != 3 ]; then | |
| sleep 10 | |
| fi | |
| done | |
| echo "Docker login failed after 3 attempts, continuing anyway" | |
| exit 0 | |
| - name: Run the container | |
| run: | | |
| set -ex | |
| echo "Starting container: aiter_test" | |
| if [ -f "/etc/podinfo/gha-render-devices" ]; then | |
| DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices) | |
| else | |
| DEVICE_FLAG="--device /dev/dri" | |
| fi | |
| IMAGE_TAG=${{ env.DOCKER_IMAGE }} | |
| docker run -dt \ | |
| --device=/dev/kfd $DEVICE_FLAG \ | |
| --shm-size=16G \ | |
| --network=host \ | |
| --group-add $(getent group render | cut -d: -f3) \ | |
| --group-add $(getent group video | cut -d: -f3) \ | |
| -e TRITON_WHEEL_DIR=/workspace/triton_wheels \ | |
| -v "${{ github.workspace }}:/workspace" \ | |
| -w /workspace \ | |
| --name aiter_test \ | |
| $IMAGE_TAG | |
| - name: Check GPU visibility | |
| run: | | |
| set -euo pipefail | |
| if docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -lc './.github/scripts/aiter_gpu_visibility_check.sh'; then | |
| echo "GPU visibility check completed" | |
| else | |
| status=$? | |
| echo "::warning::GPU visibility check failed with exit code ${status}; continuing" | |
| fi | |
| - name: Sync CK submodule | |
| run: | | |
| set -ex | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| echo "Nightly build: syncing latest CK from develop branch..." | |
| git submodule set-branch --branch develop 3rdparty/composable_kernel | |
| git submodule sync | |
| git submodule update --init --recursive --remote --jobs 4 | |
| else | |
| echo "Using pinned CK commit..." | |
| git submodule sync | |
| git submodule update --init --recursive --depth 1 --jobs 4 | |
| fi | |
| - name: Install Aiter wheel | |
| run: | | |
| set -euo pipefail | |
| ls -lh aiter_wheels | |
| shopt -s nullglob | |
| wheels=(aiter_wheels/*.whl) | |
| if [ "${#wheels[@]}" -ne 1 ]; then | |
| echo "::error::Expected exactly one Aiter wheel artifact, found ${#wheels[@]}" | |
| exit 1 | |
| fi | |
| AITER_WHEEL_PATH="/workspace/${wheels[0]}" | |
| echo "Installing Aiter wheel: ${AITER_WHEEL_PATH}" | |
| docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -lc " | |
| pip uninstall -y amd-aiter aiter || true | |
| pip install -r requirements.txt | |
| pip install --upgrade pandas pyzmq einops numpy==1.26.2 | |
| pip install --upgrade 'pybind11>=3.0.1' | |
| pip install --upgrade 'ninja>=1.11.1' | |
| pip install tabulate | |
| pip install '${AITER_WHEEL_PATH}' | |
| pip show amd-aiter | |
| " | |
| - name: Sync prebuilt kernels into source tree | |
| run: | | |
| set -euo pipefail | |
| docker exec -i \ | |
| -w /workspace \ | |
| aiter_test \ | |
| python3 - <<'PY' | |
| from importlib.metadata import distribution | |
| from pathlib import Path | |
| from shutil import copy2 | |
| dist = distribution("amd-aiter") | |
| installed_jit = Path(dist.locate_file("aiter/jit")).resolve() | |
| workspace_jit = Path("/workspace/aiter/jit").resolve() | |
| kernels = sorted(installed_jit.glob("*.so")) | |
| if not kernels: | |
| raise SystemExit(f"No prebuilt kernels found in {installed_jit}") | |
| workspace_jit.mkdir(parents=True, exist_ok=True) | |
| for kernel in kernels: | |
| copy2(kernel, workspace_jit / kernel.name) | |
| print( | |
| f"Synced {len(kernels)} prebuilt kernels from {installed_jit} " | |
| f"to {workspace_jit}" | |
| ) | |
| PY | |
| - name: Install triton | |
| run: | | |
| # Ensure install_triton.sh is available even when the PR branch | |
| # was created before the script was added to main (#2959). | |
| # The workflow file always comes from the base branch, so we | |
| # materialize the script from the base ref when missing. | |
| if [ ! -f .github/scripts/install_triton.sh ]; then | |
| BASE_REF="${{ github.event.pull_request.base.ref || github.ref_name }}" | |
| mkdir -p .github/scripts | |
| curl -fsSL \ | |
| "https://raw.githubusercontent.com/${{ github.repository }}/${BASE_REF}/.github/scripts/install_triton.sh" \ | |
| -o .github/scripts/install_triton.sh | |
| chmod +x .github/scripts/install_triton.sh | |
| fi | |
| docker exec -w /workspace aiter_test \ | |
| bash -c "./.github/scripts/install_triton.sh && pip show triton" | |
| - name: Show Aiter version | |
| run: | | |
| set -ex | |
| docker exec \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -c "pip show amd-aiter || true" | |
| - name: Tests | |
| timeout-minutes: 120 | |
| run: | | |
| set -ex | |
| docker exec \ | |
| -e MULTIGPU=TRUE \ | |
| -w /workspace \ | |
| aiter_test \ | |
| bash -c "./.github/scripts/aiter_test.sh" | |
| - name: Upload test logs | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: multigpu-test-${{ matrix.runner }} | |
| path: latest_test.log | |
| retention-days: 7 | |
| - name: Cleanup container | |
| if: always() | |
| run: | | |
| docker rm -f aiter_test || true | |
| - name: Clean up Rocm processes | |
| if: always() | |
| run: | | |
| ./.github/scripts/clean_up_rocm.sh | |
| tuned_op_bench: | |
| # Tuned operator perf regression check. | |
| # - PR: pull last main baseline CSV, compare current shard's CSV, warn-only | |
| # - push to main / workflow_dispatch: publish current CSV as next baseline | |
| # Only consumes csv from linux-aiter-mi35x-1 runner to avoid cross-arch noise. | |
| name: Tuned Op Bench | |
| if: >- | |
| always() && | |
| !cancelled() && | |
| needs.detect_aiter_test_scope.outputs.run_tests == 'true' && | |
| github.event.action != 'labeled' && | |
| github.event_name != 'schedule' | |
| runs-on: ubuntu-latest | |
| needs: [detect_aiter_test_scope, standard] | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download standard test logs (mi35x only) | |
| uses: actions/download-artifact@v4 | |
| timeout-minutes: 15 | |
| continue-on-error: true | |
| with: | |
| pattern: standard-test-log-linux-aiter-mi35x-1-shard-* | |
| path: /tmp/logs/ | |
| - name: Locate current tuned_op_bench.csv | |
| id: current | |
| run: | | |
| shopt -s nullglob | |
| csv_files=(/tmp/logs/*/tuned_op_bench.csv) | |
| if [[ ${#csv_files[@]} -eq 0 ]]; then | |
| echo "::warning::No tuned op benchmark CSV found in any mi35x shard; skipping tuned_op_bench" | |
| echo "found=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| echo "Merging tuned op benchmark CSVs:" | |
| printf ' %s\n' "${csv_files[@]}" | |
| python3 - "${csv_files[@]}" <<'PY' | |
| import csv | |
| import sys | |
| rows = [] | |
| fieldnames = [] | |
| seen = set() | |
| for path in sys.argv[1:]: | |
| with open(path, newline="") as f: | |
| reader = csv.DictReader(f) | |
| if not reader.fieldnames: | |
| continue | |
| for name in reader.fieldnames: | |
| if name not in seen: | |
| fieldnames.append(name) | |
| seen.add(name) | |
| rows.extend(reader) | |
| if "us" not in seen: | |
| raise SystemExit("merged tuned op benchmark CSV is missing required `us` column") | |
| with open("/tmp/current.csv", "w", newline="") as f: | |
| writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") | |
| writer.writeheader() | |
| for row in rows: | |
| writer.writerow({name: row.get(name, "") for name in fieldnames}) | |
| print(f"Merged {len(rows)} row(s) into /tmp/current.csv") | |
| PY | |
| echo "Current merged CSV: /tmp/current.csv ($(wc -l < /tmp/current.csv) lines)" | |
| echo "found=true" >> "$GITHUB_OUTPUT" | |
| # ── PR path: compare vs baseline ── | |
| - name: Fetch baseline from PR base.sha | |
| if: steps.current.outputs.found == 'true' && github.event_name == 'pull_request' | |
| id: baseline_pinned | |
| continue-on-error: true | |
| uses: dawidd6/action-download-artifact@v21 | |
| timeout-minutes: 15 | |
| with: | |
| workflow: aiter-test.yaml | |
| commit: ${{ github.event.pull_request.base.sha }} | |
| name: tuned-op-bench-${{ github.event.pull_request.base.sha }} | |
| path: /tmp/baseline_pinned/ | |
| # GHSA-5xr6-xhww-33m4: a fork run's artifact must never become a trusted | |
| # baseline. false is v6+'s default, stated so a flip cannot undo it. | |
| allow_forks: false | |
| if_no_artifact_found: warn | |
| - name: Fallback — fetch baseline from latest main | |
| if: >- | |
| steps.current.outputs.found == 'true' && | |
| github.event_name == 'pull_request' | |
| id: baseline_main | |
| continue-on-error: true | |
| uses: dawidd6/action-download-artifact@v21 | |
| timeout-minutes: 15 | |
| with: | |
| workflow: aiter-test.yaml | |
| branch: main | |
| name_is_regexp: true | |
| name: ^tuned-op-bench-[a-f0-9]+$ | |
| path: /tmp/baseline_main/ | |
| allow_forks: false | |
| if_no_artifact_found: warn | |
| - name: Compare | |
| if: steps.current.outputs.found == 'true' && github.event_name == 'pull_request' | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| CURR_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| set -euo pipefail | |
| baseline_csv="" | |
| if [[ -f /tmp/baseline_pinned/tuned_op_bench.csv ]]; then | |
| baseline_csv=/tmp/baseline_pinned/tuned_op_bench.csv | |
| echo "Using baseline pinned to PR.base.sha=${BASE_SHA:0:7}" | |
| else | |
| # fallback: pick first match under /tmp/baseline_main/* | |
| shopt -s nullglob | |
| candidates=(/tmp/baseline_main/*/tuned_op_bench.csv /tmp/baseline_main/tuned_op_bench.csv) | |
| for c in "${candidates[@]}"; do | |
| if [[ -f "$c" ]]; then | |
| baseline_csv="$c" | |
| echo "Using fallback baseline from latest main: $c" | |
| break | |
| fi | |
| done | |
| fi | |
| if [[ -z "$baseline_csv" ]]; then | |
| echo "::warning::No tuned op benchmark baseline found (neither pinned PR.base.sha nor latest main); skipping compare." | |
| { | |
| echo "## Tuned Op Bench" | |
| echo | |
| echo "_No baseline available — first run on this branch or main hasn't published baseline yet._" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| echo "## Tuned Op Bench (vs baseline)" >> "$GITHUB_STEP_SUMMARY" | |
| echo '```' >> "$GITHUB_STEP_SUMMARY" | |
| bash .github/scripts/check_tuned_op_regression.sh \ | |
| "$baseline_csv" /tmp/current.csv \ | |
| | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo '```' >> "$GITHUB_STEP_SUMMARY" | |
| # ── main push / workflow_dispatch path: publish baseline ── | |
| - name: Stage baseline payload | |
| if: >- | |
| steps.current.outputs.found == 'true' && | |
| (github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| || github.event_name == 'workflow_dispatch') | |
| run: | | |
| mkdir -p /tmp/publish | |
| cp /tmp/current.csv /tmp/publish/tuned_op_bench.csv | |
| python3 -c " | |
| import json, os, datetime | |
| meta = { | |
| 'commit': os.environ['GITHUB_SHA'], | |
| 'ref': os.environ['GITHUB_REF'], | |
| 'event': os.environ['GITHUB_EVENT_NAME'], | |
| 'runner_pool': 'linux-aiter-mi35x-1', | |
| 'gpu_arch_list': os.environ.get('GPU_ARCH_LIST', ''), | |
| 'ran_at': datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'), | |
| } | |
| with open('/tmp/publish/metadata.json', 'w') as f: | |
| json.dump(meta, f, indent=2) | |
| print(json.dumps(meta, indent=2)) | |
| " | |
| ls -la /tmp/publish/ | |
| wc -l /tmp/publish/tuned_op_bench.csv | |
| - name: Publish baseline artifact | |
| if: >- | |
| steps.current.outputs.found == 'true' && | |
| (github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| || github.event_name == 'workflow_dispatch') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: tuned-op-bench-${{ github.sha }} | |
| path: /tmp/publish/ | |
| retention-days: 90 | |
| aiter-test-gate: | |
| name: Aiter Test Gate | |
| if: always() | |
| runs-on: ubuntu-latest | |
| needs: | |
| - detect_aiter_test_scope | |
| - check-signal | |
| - build_aiter_wheels | |
| - split_aiter_tests | |
| - prepare_triton_wheel | |
| - standard | |
| - standard-test-finish | |
| steps: | |
| - name: Evaluate Aiter test gate | |
| env: | |
| RUN_TESTS: ${{ needs.detect_aiter_test_scope.outputs.run_tests }} | |
| REASON: ${{ needs.detect_aiter_test_scope.outputs.reason }} | |
| DETECT_RESULT: ${{ needs.detect_aiter_test_scope.result }} | |
| CHECK_SIGNAL_RESULT: ${{ needs['check-signal'].result }} | |
| BUILD_RESULT: ${{ needs.build_aiter_wheels.result }} | |
| SPLIT_RESULT: ${{ needs.split_aiter_tests.result }} | |
| TRITON_RESULT: ${{ needs.prepare_triton_wheel.result }} | |
| STANDARD_RESULT: ${{ needs.standard.result }} | |
| STANDARD_FINISH_RESULT: ${{ needs['standard-test-finish'].result }} | |
| run: | | |
| set -euo pipefail | |
| { | |
| echo "## Aiter Test Gate" | |
| echo | |
| echo "- Scope detection: ${DETECT_RESULT}" | |
| echo "- Aiter tests required: ${RUN_TESTS:-unknown}" | |
| echo "- Reason: ${REASON:-unknown}" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| if [ "${DETECT_RESULT}" != "success" ]; then | |
| echo "::error::Aiter test scope detection did not succeed: ${DETECT_RESULT}" | |
| exit 1 | |
| fi | |
| if [ "${RUN_TESTS}" != "true" ]; then | |
| echo "Aiter tests are not required for this change." >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| failed=() | |
| for result in \ | |
| "check-signal=${CHECK_SIGNAL_RESULT}" \ | |
| "build_aiter_wheels=${BUILD_RESULT}" \ | |
| "split_aiter_tests=${SPLIT_RESULT}" \ | |
| "prepare_triton_wheel=${TRITON_RESULT}" \ | |
| "standard=${STANDARD_RESULT}" \ | |
| "standard-test-finish=${STANDARD_FINISH_RESULT}" | |
| do | |
| job="${result%%=*}" | |
| status="${result#*=}" | |
| echo "- ${job}: ${status}" >> "$GITHUB_STEP_SUMMARY" | |
| if [ "${status}" != "success" ]; then | |
| failed+=("${job}=${status}") | |
| fi | |
| done | |
| if [ "${#failed[@]}" -gt 0 ]; then | |
| printf 'Aiter test gate failed because required jobs did not pass:\n' >&2 | |
| printf ' %s\n' "${failed[@]}" >&2 | |
| exit 1 | |
| fi | |
| echo "Aiter test gate passed." >> "$GITHUB_STEP_SUMMARY" |