ci: shard the Trino, Presto and Spark e2e jobs

.github/workflows/CI.yml

@@ -49,25 +49,27 @@ jobs:
     # End-to-end tests, one job per engine, in parallel: each job starts only
     # its engine's server, and runs only its engine's tests
     # (SYNALOG_E2E_ENGINES). An engine's tests run one after the other: they
-    # write the same tables.
+    # write the same tables. Trino, Presto and Spark pay a query's fixed cost
+    # on every statement, so their tests are split in three jobs each
+    # (SYNALOG_E2E_SHARD), each with its own server.
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
         include:
-          - engine: sqlite
-            service: ""
-          - engine: duckdb
-            service: ""
-          - engine: psql
-            service: postgres
-          - engine: trino
-            service: trino
-          - engine: presto
-            service: presto
-          - engine: databricks
-            service: spark
-    name: e2e (${{ matrix.engine }})
+          - { engine: sqlite, service: "", shard: "1/1" }
+          - { engine: duckdb, service: "", shard: "1/1" }
+          - { engine: psql, service: postgres, shard: "1/1" }
+          - { engine: trino, service: trino, shard: "1/3" }
+          - { engine: trino, service: trino, shard: "2/3" }
+          - { engine: trino, service: trino, shard: "3/3" }
+          - { engine: presto, service: presto, shard: "1/3" }
+          - { engine: presto, service: presto, shard: "2/3" }
+          - { engine: presto, service: presto, shard: "3/3" }
+          - { engine: databricks, service: spark, shard: "1/3" }
+          - { engine: databricks, service: spark, shard: "2/3" }
+          - { engine: databricks, service: spark, shard: "3/3" }
+    name: e2e (${{ matrix.engine }} ${{ matrix.shard }})
     steps:
       - uses: actions/checkout@v6
       - uses: dtolnay/rust-toolchain@stable
@@ -89,6 +91,7 @@ jobs:
         timeout-minutes: 180
         env:
           SYNALOG_E2E_ENGINES: ${{ matrix.engine }}
+          SYNALOG_E2E_SHARD: ${{ matrix.shard }}
           SYNALOG_E2E_REQUIRE: ${{ matrix.engine }}
           SYNALOG_E2E_PRINT_FAILURES: "1"
         run: pytest tests/e2e -q -rfE --durations=15 --timeout=600 --timeout-method=signal

docs/development.md

@@ -40,7 +40,7 @@ shell/test.sh quick        # unit + parser golden (no SQL compilation)
 
 Python steps run through `uv run`, so the project virtualenv supplies pytest and the engine drivers. The `e2e` step builds the extension into the venv and starts the server engines (PostgreSQL, Trino, Presto, and a Spark Thrift Server standing in for Databricks) itself via Docker Compose.
 
-`SYNALOG_E2E_ENGINES` runs the end-to-end tests of some engines only, a comma list (`SYNALOG_E2E_ENGINES=psql,trino uv run pytest tests/e2e`); CI runs each engine in a job of its own this way, in parallel, starting only that engine's server.
+`SYNALOG_E2E_ENGINES` runs the end-to-end tests of some engines only, a comma list (`SYNALOG_E2E_ENGINES=psql,trino uv run pytest tests/e2e`); CI runs each engine in a job of its own this way, in parallel, starting only that engine's server. `SYNALOG_E2E_SHARD=2/3` runs the second third of the selected tests, split by fixture: CI splits Trino, Presto and Spark, whose every query pays a fixed cost, into three jobs each.
 
 ## Golden test generation
 

tests/e2e/conftest.py

@@ -12,6 +12,8 @@
 
 Engine selection: SYNALOG_E2E_ENGINES=psql (a comma list) runs only the tests
 of those engines, so that CI runs each engine in a job of its own.
+SYNALOG_E2E_SHARD=2/3 runs the second third of them: CI splits the slow
+engines (Trino, Presto, Spark) into jobs that run in parallel.
 """
 
 from __future__ import annotations
@@ -20,6 +22,7 @@
 import json
 import os
 import sys
+import zlib
 from pathlib import Path
 
 import pytest
@@ -159,8 +162,32 @@ def selected_engines():
     return {e.strip() for e in os.environ.get("SYNALOG_E2E_ENGINES", "").split(",") if e.strip()}
 
 
+def selected_shard():
+    """(shard, shards) of SYNALOG_E2E_SHARD (`2/3`), 1-based; (1, 1) without it."""
+    text = os.environ.get("SYNALOG_E2E_SHARD", "").strip()
+    if not text:
+        return 1, 1
+    try:
+        shard, shards = (int(part) for part in text.split("/"))
+    except ValueError:
+        raise pytest.UsageError(f"SYNALOG_E2E_SHARD: expected a shard like 2/3, got {text!r}")
+    if not 1 <= shard <= shards:
+        raise pytest.UsageError(f"SYNALOG_E2E_SHARD: shard {shard} of {shards} does not exist")
+    return shard, shards
+
+
+def shard_of(item, shards: int) -> int:
+    """The shard (1-based) of a test: by its fixture or program, so that the
+    tests of one fixture, which share its rows, run in the same shard; the
+    same on every machine (a hash of the name, not Python's salted one)."""
+    params = getattr(getattr(item, "callspec", None), "params", {})
+    key = params.get("name") or params.get("path") or item.nodeid
+    return zlib.crc32(str(key).encode()) % shards + 1
+
+
 def pytest_configure(config):
     config.addinivalue_line("markers", "engine(name): the engine a test that takes no engine parameter runs on")
+    selected_shard()
     unknown = selected_engines() - set(ENGINES)
     if unknown:
         raise pytest.UsageError(f"SYNALOG_E2E_ENGINES: unknown engines {sorted(unknown)} (engines: {ENGINES})")
@@ -183,6 +210,12 @@ def pytest_collection_modifyitems(config, items):
     own hook of this name, to group the tests. With SYNALOG_E2E_ENGINES, the
     tests of other engines are deselected."""
     selected = selected_engines()
+    shard, shards = selected_shard()
+    if shards > 1:
+        others = [item for item in items if shard_of(item, shards) != shard]
+        if others:
+            config.hook.pytest_deselected(items=others)
+            items[:] = [item for item in items if shard_of(item, shards) == shard]
     if selected:
         deselected = [item for item in items if item_engine(item) not in selected]
         if deselected: