Size CI matrices by event to cut GitHub Actions runner minutes

.github/workflows/benchmarks.yml

@@ -6,11 +6,16 @@ on:
     types: [opened, reopened, synchronize, labeled]
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   benchmark:
     if: ${{ contains( github.event.pull_request.labels.*.name, 'run-benchmark') && github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }}
     name: Linux
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     env:
       ASV_DIR: "./asv_bench"
     defaults:

.github/workflows/ci.yml

@@ -6,6 +6,19 @@ on:
     - cron: "0 23 * * 0" # Sun 23:00 UTC
   workflow_dispatch: # allows you to trigger the workflow run manually
 
+# Runner minutes are the scarce resource here, so the matrices below are
+# deliberately event-dependent:
+#
+# * a pull request runs the *lean* matrix -- Linux only, oldest and newest
+#   supported Python. macOS bills at 10x and Windows at 2x the runner minutes
+#   of Linux, so those two platforms alone used to account for ~70% of the cost
+#   of a single CI run while catching almost nothing Linux did not.
+# * the weekly schedule and a manual ``workflow_dispatch`` run the *full*
+#   matrix: every supported Python on Linux, macOS and Windows.
+# * a pull request that needs the full matrix before merging can opt in with
+#   the ``full-ci`` label. Labels are read from the event payload, so add the
+#   label first and then re-run the workflow (or push again).
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -17,6 +30,7 @@ jobs:
   detect-ci-trigger:
     name: Detect CI trigger
     runs-on: ubuntu-slim
+    timeout-minutes: 5
     if: github.event_name == 'push' || github.event_name == 'pull_request'
     outputs:
       triggered: ${{ steps.detect-trigger.outputs.trigger-found }}
@@ -32,6 +46,7 @@ jobs:
   minimum-test-pypi:  # Runs testing suite with minimal dependencies
     name: Test minimum dependencies (Python${{ matrix.python-version }}, ${{ matrix.os }}, pip)
     runs-on: ${{ matrix.os }}-latest
+    timeout-minutes: 45
     needs: detect-ci-trigger
     # ``detect-ci-trigger`` only runs for push/pull_request, so on ``schedule``
     # and ``workflow_dispatch`` it is skipped and ``triggered`` is empty. Without
@@ -48,7 +63,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu", "macos", "windows"]
+        # See the comment at the top of the file: Linux only on pull requests,
+        # every platform on the weekly schedule, on manual dispatch and on
+        # pull requests labelled ``full-ci``.
+        os: ${{ fromJSON(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'full-ci') && '["ubuntu"]' || '["ubuntu", "macos", "windows"]') }}
         python-version: ["3.10", "3.13"]
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
@@ -58,6 +76,8 @@ jobs:
         uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
         with:
           python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
       - name: Install climpred (complete and dev dependencies)
         run: |
           python -m pip install --upgrade "pip>=25.1"
@@ -67,7 +87,16 @@ jobs:
       - name: Check dependencies
         run: |
           pip list
-      - name: Cache datasets  # pytest-xdist fails otherwise
+      - name: Locate tutorial data cache
+        id: tutorial-cache
+        run: |
+          python -c "from pooch import os_cache; print('dir=' + str(os_cache('climpred-data')))" >> "$GITHUB_OUTPUT"
+      - name: Restore tutorial datasets
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: ${{ steps.tutorial-cache.outputs.dir }}
+          key: climpred-data-${{ runner.os }}-${{ hashFiles('src/climpred/tutorial.py') }}
+      - name: Download missing tutorial datasets  # pytest-xdist fails otherwise
         run: |
           python -c "import climpred; climpred.tutorial._cache_all()"
       - name: Run tests
@@ -77,6 +106,7 @@ jobs:
   minimum-test-conda:  # Runs testing suite with minimal dependencies
     name: Test minimum dependencies (Python${{ matrix.python-version }} ubuntu, conda)
     runs-on: ubuntu-latest
+    timeout-minutes: 45
     needs: detect-ci-trigger
     # ``detect-ci-trigger`` only runs for push/pull_request, so on ``schedule``
     # and ``workflow_dispatch`` it is skipped and ``triggered`` is empty. Without
@@ -116,7 +146,16 @@ jobs:
       - name: Micromamba list
         run: |
           micromamba list
-      - name: Cache datasets  # pytest-xdist fails otherwise
+      - name: Locate tutorial data cache
+        id: tutorial-cache
+        run: |
+          python -c "from pooch import os_cache; print('dir=' + str(os_cache('climpred-data')))" >> "$GITHUB_OUTPUT"
+      - name: Restore tutorial datasets
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: ${{ steps.tutorial-cache.outputs.dir }}
+          key: climpred-data-${{ runner.os }}-${{ hashFiles('src/climpred/tutorial.py') }}
+      - name: Download missing tutorial datasets  # pytest-xdist fails otherwise
         run: |
           python -c "import climpred; climpred.tutorial._cache_all()"
       - name: Run tests
@@ -126,6 +165,7 @@ jobs:
   maximum-test-conda:  # Runs testing suite with all optional dependencies
     name: Test optional dependencies (Python${{ matrix.python-version }}, ubuntu, conda)
     runs-on: ubuntu-latest
+    timeout-minutes: 45
     needs: detect-ci-trigger
     # ``detect-ci-trigger`` only runs for push/pull_request, so on ``schedule``
     # and ``workflow_dispatch`` it is skipped and ``triggered`` is empty. Without
@@ -142,19 +182,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - env: "climpred-maximum-tests"
-            python-version: "3.10"
-            extras: "esmpy"
-          - env: "climpred-maximum-tests"
-            python-version: "3.11"
-            extras: "esmpy"
-          - env: "climpred-maximum-tests"
-            python-version: "3.12"
-            extras: "esmpy"
-          - env: "climpred-maximum-tests"
-            python-version: "3.13"
-            extras: "esmpy"
+        # Oldest and newest supported Python on pull requests, the full range
+        # on the weekly schedule, on manual dispatch and on ``full-ci`` PRs.
+        python-version: ${{ fromJSON(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'full-ci') && '["3.10", "3.13"]' || '["3.10", "3.11", "3.12", "3.13"]') }}
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
@@ -165,31 +195,44 @@ jobs:
           environment-file: ci/requirements/maximum-tests.yml
           environment-name: climpred-maximum-tests
           cache-environment: true
-          cache-environment-key: "ubuntu-latest-${{ runner.arch }}-py${{ matrix.python-version }}-${{ env.TODAY }}-${{ hashFiles(env.CONDA_ENV_FILE) }}"
           create-args: >-
             python=${{ matrix.python-version }}
-            ${{ matrix.extras }}
+            esmpy
       - name: Install climpred (no deps) + remove netcdf4
         run: |
           python -m pip install --no-deps -e .
           python -m pip uninstall --yes netcdf4
-      - name: Install bleeding edge
-        if: matrix.env == 'climpred-maximum-tests-upstream'
-        run: |
-          python -m pip install -r requirements_upstream.txt
       - name: Micromamba info
         run: |
           micromamba info
       - name: Micromamba list
         run: |
           micromamba list
-      - name: Cache datasets  # pytest-xdist fails otherwise
+      - name: Locate tutorial data cache
+        id: tutorial-cache
+        run: |
+          python -c "from pooch import os_cache; print('dir=' + str(os_cache('climpred-data')))" >> "$GITHUB_OUTPUT"
+      - name: Restore tutorial datasets
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: ${{ steps.tutorial-cache.outputs.dir }}
+          key: climpred-data-${{ runner.os }}-${{ hashFiles('src/climpred/tutorial.py') }}
+      - name: Download missing tutorial datasets  # pytest-xdist fails otherwise
         run: |
           python -c "import climpred; climpred.tutorial._cache_all()"
+      # Coverage instrumentation slows the suite down measurably and four
+      # identical reports tell us nothing the newest Python alone does not, so
+      # only that entry measures and uploads coverage.
       - name: Run tests
+        if: matrix.python-version != '3.13'
+        run: |
+          pytest -n auto --durations=20
+      - name: Run tests with coverage
+        if: matrix.python-version == '3.13'
         run: |
           pytest -n auto --durations=20 --cov=climpred --cov-report=xml
       - name: Upload coverage to codecov
+        if: matrix.python-version == '3.13'
         uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v7.0.0
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
@@ -199,6 +242,7 @@ jobs:
   doctest:  # tests all docstrings
     name: Doctests (Python${{ matrix.python-version }}, ubuntu)
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     defaults:
       run:
         shell: bash -l {0}
@@ -224,13 +268,23 @@ jobs:
       - name: Micromamba list
         run: |
           micromamba list
+      - name: Locate tutorial data cache
+        id: tutorial-cache
+        run: |
+          python -c "from pooch import os_cache; print('dir=' + str(os_cache('climpred-data')))" >> "$GITHUB_OUTPUT"
+      - name: Restore tutorial datasets
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: ${{ steps.tutorial-cache.outputs.dir }}
+          key: climpred-data-${{ runner.os }}-${{ hashFiles('src/climpred/tutorial.py') }}
       - name: Run doctests
         run: |
           python -m pytest --doctest-modules src/climpred --ignore src/climpred/tests
 
   notebooks:  # Checks that pre-compiled notebooks in docs still work.
     name: Notebooks (Python${{ matrix.python-version }}, ubuntu)
     runs-on: ubuntu-latest
+    timeout-minutes: 45
     defaults:
         run:
           shell: bash -l {0}
@@ -256,6 +310,15 @@ jobs:
       - name: Micromamba list
         run: |
           micromamba list
+      - name: Locate tutorial data cache
+        id: tutorial-cache
+        run: |
+          python -c "from pooch import os_cache; print('dir=' + str(os_cache('climpred-data')))" >> "$GITHUB_OUTPUT"
+      - name: Restore tutorial datasets
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: ${{ steps.tutorial-cache.outputs.dir }}
+          key: climpred-data-${{ runner.os }}-${{ hashFiles('src/climpred/tutorial.py') }}
       - name: Test notebooks in docs
         run: |
           pushd docs

.github/workflows/upstream-dev.yml

@@ -19,6 +19,7 @@ jobs:
   detect-ci-trigger:
     name: Detect upstream-dev CI trigger
     runs-on: ubuntu-slim
+    timeout-minutes: 5
     if: github.event_name == 'push' || github.event_name == 'pull_request'
     outputs:
       triggered: ${{ steps.detect-trigger.outputs.trigger-found }}
@@ -34,6 +35,7 @@ jobs:
   upstream-dev:
     name: upstream-dev (Python${{ matrix.python-version }}, ubuntu)
     runs-on: ubuntu-latest
+    timeout-minutes: 45
     needs: detect-ci-trigger
     if: |
       always()
@@ -110,6 +112,7 @@ jobs:
       && github.repository == 'pangeo-data/climpred'
       && needs.upstream-dev.outputs.artifacts_availability == 'true'
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     defaults:
       run:
         shell: bash

AGENTS.md

@@ -60,6 +60,24 @@ gh run list --limit 5
 gh run watch <RUN_ID>
 ```
 
+## Continuous Integration
+
+Runner minutes are treated as a scarce resource, so `ci.yml` sizes its matrices
+by event:
+
+- **Pull requests** run a lean matrix: Linux only, Python 3.10 and 3.13. macOS
+  bills at 10x and Windows at 2x Linux runner minutes.
+- **The weekly schedule and `workflow_dispatch`** run the full matrix: Linux,
+  macOS and Windows, Python 3.10 through 3.13.
+- **A pull request that needs the full matrix before merging** can opt in with
+  the `full-ci` label. Labels are read from the event payload, so add the label
+  first and then re-run the workflow (or push again).
+- Coverage is measured and uploaded by the Python 3.13 `maximum-test-conda`
+  entry only.
+- `[skip-ci]` in a commit message skips the test jobs; `[test-upstream]` opts a
+  pull request into `upstream-dev.yml`; the `run-benchmark` label opts it into
+  `benchmarks.yml`.
+
 ## Code Style & Conventions
 
 - **Formatter**: Black (line-length 88)

CHANGELOG.rst

@@ -30,6 +30,8 @@ Internals/Minor Fixes
 - :py:meth:`.HindcastEnsemble.smooth` and :py:meth:`.PerfectModelEnsemble.smooth` propagate the ``lead`` ``units`` attribute onto the ``lead_center`` coordinate that is added during verification of temporally smoothed ensembles. `Aaron Spring`_
 - :py:meth:`.HindcastEnsemble.plot_alignment` now raises a clear error message when there is no overlap between hindcast ``valid_time`` and verification ``time``, instead of a cryptic ``ValueError`` about ``CFTimeIndex`` ambiguity. (:issue:`912`, :pr:`921`) `Aaron Spring`_
 - Restored GitHub Actions CI, which had been disabled for repository inactivity. Both workflows are re-registered as ``ci.yml`` and ``upstream-dev.yml``, scheduled and manual runs no longer skip the test matrix, and the upstream test runs on Python 3.13. (:pr:`929`) `Aaron Spring`_
+- Reduced GitHub Actions runner-minute usage. Pull requests now run a lean test matrix (Linux, Python 3.10 and 3.13); the full matrix across macOS, Windows and Python 3.10-3.13 runs on the weekly schedule, on manual dispatch and on pull requests labelled ``full-ci``. Coverage is measured and uploaded once instead of four times, tutorial datasets and ``pip`` wheels are cached between runs, and every job has a ``timeout-minutes`` cap. The ``maximum-test-conda`` job also dropped a hand-rolled ``cache-environment-key`` that interpolated two undefined variables; ``setup-micromamba`` already appends a hash of the environment file, the ``create-args``, the environment name and the OS to the cache key. (:pr:`931`) `Aaron Spring`_
+- Fixed ``tool.pytest.ini_options.testpaths``, which still pointed at the pre-src-layout ``climpred/tests``. `pytest` found nothing there and silently rescanned the whole repository on every run. (:pr:`931`) `Aaron Spring`_
 - Updated the :py:meth:`.HindcastEnsemble.remove_bias` doctest values for ``how="modified_quantile"`` to match current `bias_correction` output, fixing the ``Doctests`` CI job. (:pr:`930`) `Aaron Spring`_
 
 

pyproject.toml

@@ -160,7 +160,7 @@ exclude = [ "asv_bench/", "docs/", "debug_notebooks/" ]
 [tool.pytest.ini_options]
 python_files = [ "test_*.py" ]
 addopts = [ "--color=yes", "--verbose" ]
-testpaths = [ "climpred/tests" ]
+testpaths = [ "src/climpred/tests" ]
 filterwarnings = [
   # xarray
   "ignore: Using a non-tuple sequence for multidimensional indexing is deprecated:FutureWarning",