gauntlet.yml

# Validation Gauntlet: gold-labelled disposition benchmark for the validation,
# domain and text-cleaning surfaces. Runs the lightweight fixtures on every PR
# and gates on the absolute thresholds plus no-regression vs the stored
# baseline (benchmarks/gauntlet/baseline.json). Heavier sizes stay manual.
name: Validation Gauntlet

on:
  pull_request:
    paths-ignore:
      - "docs/**"
      - "*.md"
  workflow_dispatch:
    inputs:
      rows:
        description: "rows per fixture"
        default: "300"
      update_baseline:
        description: "re-pin baseline.json from this run (commit it manually)"
        type: boolean
        default: false

permissions:
  contents: read

jobs:
  gauntlet:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
          cache: pip
      - name: Install
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[dev]"
      - name: Run gauntlet with gates
        run: |
          ROWS="${{ github.event.inputs.rows || '300' }}"
          EXTRA=""
          if [ "${{ github.event.inputs.update_baseline }}" = "true" ]; then
            EXTRA="--update-baseline"
          fi
          python -m benchmarks.gauntlet run --rows "$ROWS" --check $EXTRA
      - name: Job summary
        if: always()
        run: |
          if [ -f benchmarks/gauntlet/results/gauntlet.md ]; then
            cat benchmarks/gauntlet/results/gauntlet.md >> "$GITHUB_STEP_SUMMARY"
          fi
      - name: Upload results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: gauntlet-results
          path: benchmarks/gauntlet/results/
          if-no-files-found: warn