python-dist.yml

name: Python Distributions

# The release's Python build: every wheel leg and the sdist, then each baseline
# wheel and the sdist installed and smoke-tested. release.yml calls this on a
# tag and uploads what it built; release-rehearsal.yml calls it and publishes
# nothing. Build steps live only here, so the two cannot drift apart.

on:
  workflow_call:
    inputs:
      sccache:
        description: Cache compiler output with sccache
        type: boolean
        default: true

permissions:
  contents: read

env:
  CARGO_INCREMENTAL: 0
  CARGO_NET_RETRY: 10
  RUST_BACKTRACE: short
  RUSTUP_MAX_RETRIES: 10

jobs:
  python-interpreters:
    name: Python Interpreter Contract
    runs-on: ubuntu-latest
    outputs:
      setup-versions: ${{ steps.contract.outputs.setup-versions }}
    steps:
      - uses: actions/checkout@v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-python@v7
        with:
          python-version: '3.14'
      - name: Validate metadata and generate build interpreters
        id: contract
        run: python .github/scripts/wheel_interpreters.py --github-output "$GITHUB_OUTPUT"

  # Build Python wheels for the base public package API.
  python-wheels:
    name: Build Python Wheels (${{ matrix.os }}-${{ matrix.target }}-${{ matrix.cpu_profile }})
    runs-on: ${{ matrix.os }}
    needs: python-interpreters
    strategy:
      # One runner failing (sccache server startup on macos-15-intel timed out
      # during the first v0.9.0 attempt) must not cancel the other eight wheels.
      # Let them finish so a re-run only has to retry the one that broke.
      fail-fast: false
      matrix:
        include:
          # Baseline compatibility builds (maximum compatibility)
          - os: ubuntu-latest
            target: x86_64
            cpu_profile: baseline
            rustflags: "-C target-cpu=x86-64 -C target-feature=-avx,-avx2,-avx512f -C opt-level=2"
          - os: ubuntu-latest
            target: aarch64
            cpu_profile: baseline
            rustflags: "-C target-cpu=generic -C opt-level=2"
            # manylinux2014's aarch64 cross image ships GCC 4.8.5, which does
            # not define __ARM_ARCH; ring's assembly refuses to build without
            # it and upstream closed that WONTFIX (briansmith/ring#1728).
            # 2_28 ships GCC 7.5. Raises this arch's glibc floor 2.17 -> 2.28.
            manylinux: '2_28'
          - os: windows-latest
            target: x64
            cpu_profile: baseline
            rustflags: "-C target-cpu=x86-64 -C target-feature=-avx,-avx2,-avx512f -C opt-level=2"
          - os: macos-15-intel
            target: x86_64
            cpu_profile: baseline
            rustflags: "-C target-cpu=x86-64 -C target-feature=-avx,-avx2,-avx512f -C opt-level=2"
          - os: macos-latest
            target: aarch64
            cpu_profile: baseline
            # No -C target-cpu: `generic` strips neon/aes/sha2/pmull, which
            # ring asserts are statically present on aarch64-apple-darwin
            # (src/cpu/arm/darwin.rs). Every arm64 Mac has them, so `generic`
            # bought no compatibility and cost performance.
            rustflags: "-C opt-level=2"

          # Optimized builds (performance optimized)
          - os: ubuntu-latest
            target: x86_64
            cpu_profile: optimized
            rustflags: "-C target-cpu=native"
          - os: windows-latest
            target: x64
            cpu_profile: optimized
            rustflags: "-C target-cpu=native"
          - os: macos-15-intel
            target: x86_64
            cpu_profile: optimized
            rustflags: "-C target-cpu=native"
          - os: macos-latest
            target: aarch64
            cpu_profile: optimized
            # The same ring constraint as the baseline leg above, from the
            # other side. ring asserts `CAPS_STATIC == MIN_STATIC_FEATURES`
            # exactly on aarch64-apple-darwin (src/cpu/arm/darwin.rs), so
            # `native` fails on a current Apple Silicon runner by enabling
            # more than that baseline, just as `generic` failed by enabling
            # less. rustc already defaults this target to apple-m1, which is
            # what `native` was reaching for, so dropping it costs nothing.
            rustflags: "-C opt-level=3"

    steps:
    - uses: actions/checkout@v7.0.1
      with:
        persist-credentials: false
    - uses: actions/setup-python@v7
      with:
        # Install every declared version on Windows/macOS. Linux builds run
        # inside manylinux, where maturin selects the explicit names below.
        python-version: ${{ needs.python-interpreters.outputs.setup-versions }}

    - name: Install uv for host interpreter discovery
      if: runner.os != 'Linux'
      uses: astral-sh/setup-uv@v7

    - name: Select build interpreters
      id: build-interpreters
      # Linux needs names inside manylinux; Windows/macOS use installed host paths.
      run: python .github/scripts/wheel_interpreters.py --github-output "$GITHUB_OUTPUT" ${{ runner.os != 'Linux' && '--resolve-interpreters' || '' }}
      shell: bash

    - name: Read the pinned toolchain
      id: toolchain
      shell: bash
      run: |
        channel=$(sed -n 's/^channel *= *"\(.*\)"$/\1/p' rust-toolchain.toml)
        if [ -z "$channel" ]; then
          echo "::error::no channel in rust-toolchain.toml"
          exit 1
        fi
        echo "channel=$channel" >> "$GITHUB_OUTPUT"

    - name: Build wheels (${{ matrix.cpu_profile }})
      uses: PyO3/maturin-action@v1
      with:
        target: ${{ matrix.target }}
        # The shipped feature set is declared once, in `[tool.maturin] features`
        # in pyproject.toml; building without --features keeps the wheel and the
        # PEP 517 sdist build from silently diverging (see issue #589).
        args: --release --out dist --interpreter ${{ steps.build-interpreters.outputs.interpreters }}
        sccache: ${{ inputs.sccache }}
        manylinux: ${{ matrix.manylinux || 'auto' }}
        # Ensure compatibility flags are properly set
        rust-toolchain: ${{ steps.toolchain.outputs.channel }}
      env:
        RUSTFLAGS: ${{ matrix.rustflags }}
        # Additional environment variables to ensure compatibility for all targets
        CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: ${{ matrix.rustflags }}
        CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS: ${{ matrix.rustflags }}
        CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS: ${{ matrix.rustflags }}
        CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS: ${{ matrix.rustflags }}
        CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS: ${{ matrix.rustflags }}
        # Disable CPU-specific optimizations in C dependencies (Linux/macOS x86_64 only)
        CFLAGS: ${{ matrix.target == 'x86_64' && '-march=x86-64' || '' }}
        CXXFLAGS: ${{ matrix.target == 'x86_64' && '-march=x86-64' || '' }}

    - name: Verify published interpreter and ABI set
      run: python .github/scripts/wheel_interpreters.py --wheels dist
      shell: bash

    - name: Verify wheel compatibility (baseline only)
      if: matrix.cpu_profile == 'baseline'
      run: |
        # Install objdump/readelf for analysis (Linux only for now)
        if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
          echo "Verifying baseline wheel has no advanced CPU instructions..."
          cd dist

          for wheel in *.whl; do
            if [ -f "$wheel" ]; then
              echo "Analyzing wheel: $wheel"

              # Extract wheel and find .so files
              python -m zipfile -e "$wheel" wheel_extracted/

              # Check for advanced CPU instructions in shared libraries
              find wheel_extracted/ -name "*.so" -exec sh -c '
                echo "Checking $1 for advanced CPU instructions..."
                if objdump -d "$1" 2>/dev/null | grep -E "(vzeroupper|vmovdqu|vpor|vpaddd)" >/dev/null; then
                  echo "WARNING: Found AVX instructions in $1"
                  echo "This wheel may not be compatible with older CPUs"
                else
                  echo "OK: No advanced CPU instructions found in $1"
                fi
              ' _ {} \;

              rm -rf wheel_extracted/
            fi
          done
        else
          echo "Skipping compatibility check on ${{ matrix.os }}"
        fi
      shell: bash

    - name: Rename wheels with CPU profile suffix
      run: |
        cd dist
        for wheel in *.whl; do
          if [ -f "$wheel" ]; then
            # Only rename optimized wheels - baseline wheels keep standard names for PyPI
            if [ "${{ matrix.cpu_profile }}" = "optimized" ]; then
              base_name="${wheel%.whl}"
              new_name="${base_name}-optimized.whl"
              mv "$wheel" "$new_name"
              echo "Renamed optimized wheel: $wheel -> $new_name"
            else
              echo "Keeping baseline wheel name: $wheel"
            fi
          fi
        done
      shell: bash

    - name: Upload wheels as artifacts
      uses: actions/upload-artifact@v7
      with:
        name: wheels-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.cpu_profile }}
        path: dist

  # Build an sdist so unsupported environments still have an install path.
  python-sdist:
    name: Build Python sdist
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v7.0.1
      with:
        persist-credentials: false
    - uses: actions/setup-python@v7
      with:
        python-version: '3.11'

    - name: Read the pinned toolchain
      id: toolchain
      shell: bash
      run: |
        channel=$(sed -n 's/^channel *= *"\(.*\)"$/\1/p' rust-toolchain.toml)
        if [ -z "$channel" ]; then
          echo "::error::no channel in rust-toolchain.toml"
          exit 1
        fi
        echo "channel=$channel" >> "$GITHUB_OUTPUT"

    - name: Build sdist
      uses: PyO3/maturin-action@v1
      with:
        command: sdist
        args: --out dist
        manylinux: auto
        rust-toolchain: ${{ steps.toolchain.outputs.channel }}

    - name: Upload sdist as artifact
      uses: actions/upload-artifact@v7
      with:
        name: python-sdist
        path: dist/*.tar.gz

  # A wheel that builds can still fail to install or import where it runs, and
  # the aarch64 Linux leg is cross-compiled, so before this job nothing ran its
  # wheels before users did. Each baseline wheel goes into a bare venv on its
  # own platform, one interpreter at a time, and runs the wheel smoke test,
  # which also checks the wheel names the commit it was built from. Optimized
  # wheels target the build runner's CPU, which another runner need not share,
  # so they are not installed here.
  wheel-install:
    name: Install Wheels (${{ matrix.runner }})
    runs-on: ${{ matrix.runner }}
    needs: [python-interpreters, python-wheels]
    strategy:
      fail-fast: false
      matrix:
        include:
          - runner: ubuntu-latest
            artifact: wheels-ubuntu-latest-x86_64-baseline
          - runner: ubuntu-24.04-arm
            artifact: wheels-ubuntu-latest-aarch64-baseline
          - runner: windows-latest
            artifact: wheels-windows-latest-x64-baseline
          - runner: macos-15-intel
            artifact: wheels-macos-15-intel-x86_64-baseline
          - runner: macos-latest
            artifact: wheels-macos-latest-aarch64-baseline
    defaults:
      run:
        shell: bash

    steps:
    - uses: actions/checkout@v7.0.1
      with:
        persist-credentials: false
    - uses: actions/setup-python@v7
      with:
        python-version: ${{ needs.python-interpreters.outputs.setup-versions }}

    - name: Install uv for interpreter discovery
      uses: astral-sh/setup-uv@v7

    - name: Download this platform's baseline wheels
      uses: actions/download-artifact@v8
      with:
        name: ${{ matrix.artifact }}
        path: dist

    - name: Install and smoke-test each interpreter's wheel
      env:
        # Every job of a run checks out the same commit, so each wheel must
        # name it.
        DATAPROF_EXPECTED_COMMIT: ${{ github.sha }}
        # One minor version per line.
        VERSIONS: ${{ needs.python-interpreters.outputs.setup-versions }}
      run: |
        cp .github/scripts/wheel_smoke.py "$RUNNER_TEMP/wheel_smoke.py"
        failed=0
        {
          echo "### ${{ matrix.artifact }}"
          echo ""
          echo "| Interpreter | Wheel | Installs | Smoke test |"
          echo "|---|---|---|---|"
        } >> "$GITHUB_STEP_SUMMARY"
        for version in $VERSIONS; do
          tag="cp${version/./}"
          wheel=$(find dist -maxdepth 1 -name "dataprof-*-${tag}-${tag}-*.whl" | head -n 1)
          built="missing"
          installs="not run"
          smoke="not run"
          if [ -n "$wheel" ]; then
            built="$(basename "$wheel")"
            python=$(uv python find --system --no-python-downloads "$version")
            venv="$RUNNER_TEMP/venv-$tag"
            "$python" -m venv "$venv"
            venv_python="$venv/bin/python"
            [ -x "$venv_python" ] || venv_python="$venv/Scripts/python.exe"
            echo "::group::CPython $version: $built"
            # --no-index: the base wheel declares no dependencies.
            if "$venv_python" -m pip install --no-index "$wheel"; then
              installs="yes"
              # From outside the checkout, so the import cannot resolve to python/.
              if (cd "$RUNNER_TEMP" && "$venv_python" wheel_smoke.py); then
                smoke="passed"
              else
                smoke="failed"
              fi
            else
              installs="failed"
            fi
            echo "::endgroup::"
          fi
          echo "| $version | $built | $installs | $smoke |" >> "$GITHUB_STEP_SUMMARY"
          if [ "$smoke" != "passed" ]; then
            echo "::error::CPython $version on ${{ matrix.runner }}: wheel $built, installs $installs, smoke test $smoke"
            failed=1
          fi
        done
        exit "$failed"

  # The sdist is the install path everywhere no wheel fits. Build and install
  # it the way pip does there, then run the same smoke test. It carries no git
  # metadata, so the package built from it records no commit.
  sdist-install:
    name: Install sdist
    runs-on: ubuntu-latest
    needs: python-sdist
    defaults:
      run:
        shell: bash

    steps:
    - uses: actions/checkout@v7.0.1
      with:
        persist-credentials: false
    - uses: actions/setup-python@v7
      with:
        python-version: '3.11'

    - name: Download sdist
      uses: actions/download-artifact@v8
      with:
        name: python-sdist
        path: dist

    - name: Build, install and smoke-test the sdist
      run: |
        python -m venv "$RUNNER_TEMP/venv"
        "$RUNNER_TEMP/venv/bin/python" -m pip install dist/*.tar.gz
        cp .github/scripts/wheel_smoke.py "$RUNNER_TEMP/wheel_smoke.py"
        cd "$RUNNER_TEMP"
        "$RUNNER_TEMP/venv/bin/python" wheel_smoke.py