ci(rust): skip unnecessary setup steps

.github/actions/rust/pre-merge/action.yml

@@ -20,7 +20,7 @@ description: Rust pre-merge testing and linting github iggy actions
 
 inputs:
   task:
-    description: "Task to run (check, check-msrv, fmt, clippy, sort, machete, doctest, verify-publish, test-1, test-2, compat, miri)"
+    description: "Task to run (check, check-msrv, fmt, clippy, sort, machete, doctest, verify-publish, test-build, test-run-1, test-run-2, compat, miri)"
     required: true
   component:
     description: "Component name (for context)"
@@ -33,6 +33,12 @@ runs:
     - name: Setup Rust with cache
       uses: ./.github/actions/utils/setup-rust-with-cache
       with:
+        # Only test jobs execute nextest. Avoid downloading it in lint, metadata,
+        # and cross-build jobs.
+        install-nextest: ${{ inputs.task == 'test-build' || startsWith(inputs.task, 'test-run-') }}
+        # These tasks do not compile workspace crates and need no native
+        # libraries. All compiling tasks keep the existing dependency setup.
+        install-system-dependencies: ${{ inputs.task != 'fmt' && inputs.task != 'sort' && inputs.task != 'machete' }}
         # Miri builds against a nightly toolchain with a separate `target/miri`
         # subtree; isolate its cache from the stable `dev` namespace so the
         # two don't evict each other.
@@ -45,7 +51,7 @@ runs:
         # fmt/sort/machete never build into target/, so the multi-GB cache
         # restore is pure overhead. Compiling legs (check/clippy/doctest/test-*)
         # keep it; a cold cache there recompiles the whole dep tree.
-        read-cache: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'machete') && 'false' || 'true' }}
+        read-cache: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'machete' || startsWith(inputs.task, 'test-run-')) && 'false' || 'true' }}
         # Light legs fit in the runner's ~89 GiB default headroom; skip the
         # ~20-45s reclaim. Disk-heavy legs (coverage build + testcontainers
         # images on test-*, cross-builds, miri, verify-publish) keep it.
@@ -79,18 +85,18 @@ runs:
     # Safety: cargo check/clippy run on the full workspace separately, catching all
     # compilation errors. This only scopes test BUILD and EXECUTION.
     - name: Fetch base branch for DAG analysis
-      if: startsWith(inputs.task, 'test-')
+      if: inputs.task == 'test-build'
       run: git fetch origin master --depth=1 2>/dev/null || true
       shell: bash
 
     - name: Install cargo-rail
-      if: startsWith(inputs.task, 'test-')
+      if: inputs.task == 'test-build'
       uses: taiki-e/install-action@v2
       with:
         tool: cargo-rail
 
     - name: Compute affected crates (cargo-rail)
-      if: startsWith(inputs.task, 'test-')
+      if: inputs.task == 'test-build'
       run: |
         METADATA_JSON=$(cargo metadata --format-version 1 --no-deps 2>/dev/null || echo "{}")
         TOTAL_CRATES=$(echo "$METADATA_JSON" | jq '.workspace_members | length' 2>/dev/null || echo "?")
@@ -110,8 +116,6 @@ runs:
             CRATE_COUNT=$(echo "$CRATES" | wc -l)
             # Build nextest filter expression for cargo nextest run (affected crates only)
             echo "$CRATES" | sed 's/^/package(/; s/$/)/' | paste -sd '|' | sed 's/|/ | /g' > /tmp/nextest-filter.txt
-            # Save affected-only -p flags for cargo test fallback (no nextest filter)
-            echo "$CRATES" | sed 's/^/-p /' | tr '\n' ' ' > /tmp/test-packages.txt
             # Build -p flags: affected crates + packages with bin/cdylib targets.
             # Binary packages: nextest sets CARGO_BIN_EXE_<name> from compiled
             # artifacts; cdylib packages: connector plugins loaded via dlopen.
@@ -187,44 +191,30 @@ runs:
       shell: bash
 
     - name: Install dependencies for Rust tests
-      if: startsWith(inputs.task, 'test-') && runner.os == 'Linux'
+      if: startsWith(inputs.task, 'test-run-') && runner.os == 'Linux'
       run: |
         sudo apt-get install --yes musl-tools gnome-keyring keyutils dbus-x11 libsecret-tools
         rm -f $HOME/.local/share/keyrings/*
       shell: bash
 
     - name: Install cargo-llvm-cov
-      if: startsWith(inputs.task, 'test-')
+      if: inputs.task == 'test-build' || startsWith(inputs.task, 'test-run-')
       uses: taiki-e/install-action@v2
       with:
         tool: cargo-llvm-cov
 
-    - name: Build and test with coverage
-      if: startsWith(inputs.task, 'test-')
+    - name: Build and archive tests with coverage
+      if: inputs.task == 'test-build'
       run: |
-        # Parse partition index from task name (test-1 -> hash:1/2, test-2 -> hash:2/2)
-        TASK="${{ inputs.task }}"
-        PARTITION_FLAG=""
-        if [[ "$TASK" =~ ^test-([0-9]+)$ ]]; then
-          PARTITION_INDEX="${BASH_REMATCH[1]}"
-          PARTITION_FLAG="--partition hash:${PARTITION_INDEX}/2"
-          echo "::notice::Running test partition ${PARTITION_INDEX}/2"
-        fi
-
-        # Read DAG-based affected crate filter (computed in earlier step)
         NEXTEST_FILTER=""
         PACKAGE_FLAGS=""
-        TEST_PACKAGE_FLAGS=""
         TOTAL_CRATES="?"
         if [[ -f /tmp/nextest-filter.txt ]]; then
           NEXTEST_FILTER=$(cat /tmp/nextest-filter.txt)
         fi
         if [[ -f /tmp/packages.txt ]]; then
           PACKAGE_FLAGS=$(cat /tmp/packages.txt)
         fi
-        if [[ -f /tmp/test-packages.txt ]]; then
-          TEST_PACKAGE_FLAGS=$(cat /tmp/test-packages.txt)
-        fi
         if [[ -f /tmp/total-crates.txt ]]; then
           TOTAL_CRATES=$(cat /tmp/total-crates.txt)
         fi
@@ -237,17 +227,7 @@ runs:
           echo "::notice::Full workspace build (no DAG filter available)"
         fi
 
-        source <(cargo llvm-cov show-env --export-prefix)
-
-        # Doris 4.0.3's start_be.sh hard-`exit 1`s unless vm.max_map_count >= 2000000.
-        # This kernel param can only be set on the host (no container can raise it),
-        # so we raise it here. Everything else about booting Doris — image, heap/mem
-        # caps, port mappings, BE-alive wait — lives in the testcontainers fixture
-        # at core/integration/tests/connectors/fixtures/doris/container.rs, so
-        # `cargo test` and CI follow exactly the same path.
-        if [[ "$RUNNER_OS" == "Linux" ]]; then
-          sudo sysctl -w vm.max_map_count=2000000 || true
-        fi
+        source <(cargo llvm-cov show-env --sh)
 
         bins_start=$(date +%s)
         if [[ -n "$PACKAGE_FLAGS" ]]; then
@@ -259,52 +239,65 @@ runs:
         bins_duration=$((bins_end - bins_start))
         echo "::notice::Binaries and libraries built in ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))"
 
+        ARTIFACT_DIR="${RUNNER_TEMP}/rust-nextest-archive"
+        mkdir -p "$ARTIFACT_DIR"
+
         compile_start=$(date +%s)
         if [[ -n "$PACKAGE_FLAGS" ]]; then
-          cargo test --locked --no-run $PACKAGE_FLAGS
+          cargo nextest archive --locked $PACKAGE_FLAGS \
+            --archive-file "$ARTIFACT_DIR/rust-tests.tar.zst"
         else
-          cargo test --locked --no-run
+          cargo nextest archive --locked \
+            --archive-file "$ARTIFACT_DIR/rust-tests.tar.zst"
         fi
         compile_end=$(date +%s)
         compile_duration=$((compile_end - compile_start))
-        echo "::notice::Tests compiled in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))"
+        echo "::notice::Tests compiled and archived in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))"
 
-        # Start D-Bus and unlock keyring right before test execution to avoid
-        # gnome-keyring auto-locking the collection during the build phase.
-        # Previously this ran before `cargo build`, leaving a 7+ minute idle
-        # window that triggered org.freedesktop.Secret.Error.IsLocked ~10% of runs.
-        if [[ "$RUNNER_OS" == "Linux" ]]; then
-          eval $(dbus-launch --sh-syntax)
-          export DBUS_SESSION_BUS_ADDRESS
-          eval $(echo -n "test" | gnome-keyring-daemon --unlock --components=secrets)
-          echo -n "warmup" | secret-tool store --label="ci-warmup" ci-test warmup
+        if [[ -n "$NEXTEST_FILTER" ]]; then
+          printf '%s\n' "$NEXTEST_FILTER" > "$ARTIFACT_DIR/nextest-filter.txt"
+        else
+          printf '%s\n' 'all()' > "$ARTIFACT_DIR/nextest-filter.txt"
         fi
 
-        test_start=$(date +%s)
-        if command -v cargo-nextest &> /dev/null; then
-          if [[ -n "$NEXTEST_FILTER" ]]; then
-            cargo nextest run --locked --no-fail-fast --profile ci $PARTITION_FLAG $PACKAGE_FLAGS -E "$NEXTEST_FILTER"
-          else
-            cargo nextest run --locked --no-fail-fast --profile ci $PARTITION_FLAG
-          fi
+        PLUGIN_FILES=()
+        while IFS= read -r -d '' plugin_file; do
+          PLUGIN_FILES+=("${plugin_file##*/}")
+        done < <(
+          find target/debug -maxdepth 1 -type f \
+            -name 'libiggy_connector_*.so' -print0
+        )
+        if (( ${#PLUGIN_FILES[@]} > 0 )); then
+          tar -C target/debug -cf "$ARTIFACT_DIR/connector-plugins.tar" \
+            "${PLUGIN_FILES[@]}"
         else
-          if [[ -n "$PARTITION_FLAG" ]]; then
-            echo "::error::cargo-nextest not found, falling back to cargo test without partitioning (all tests will run on every partition)"
-          fi
-          # Use TEST_PACKAGE_FLAGS (affected crates only), not PACKAGE_FLAGS
-          # (which includes binary packages whose tests should NOT run).
-          if [[ -n "$TEST_PACKAGE_FLAGS" ]]; then
-            cargo test --locked --no-fail-fast $TEST_PACKAGE_FLAGS
-          else
-            cargo test --locked --no-fail-fast
-          fi
+          echo "::error::No connector plugin libraries found in target/debug"
+          exit 1
         fi
-        test_end=$(date +%s)
-        test_duration=$((test_end - test_start))
-        echo "::notice::Tests executed in ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))"
+
+        # The integration package launches binaries owned by other workspace
+        # packages through assert_cmd::cargo_bin. Nextest only exposes
+        # CARGO_BIN_EXE_* for binaries Cargo associates with the current
+        # integration-test package. Preserve these cross-package executables
+        # explicitly so test-run jobs can provide the expected paths.
+        RUNTIME_BINARIES=(
+          iggy
+          iggy-server
+          iggy-server-ng
+          iggy-connectors
+          iggy-mcp
+          iggy-bench
+        )
+        for binary in "${RUNTIME_BINARIES[@]}"; do
+          if [[ ! -x "target/debug/${binary}" ]]; then
+            echo "::error::Required test runtime binary not found: target/debug/${binary}"
+            exit 1
+          fi
+        done
+        tar -C target/debug -czf "$ARTIFACT_DIR/runtime-binaries.tar.gz" \
+          "${RUNTIME_BINARIES[@]}"
 
         build_duration=$((bins_duration + compile_duration))
-        total_duration=$((build_duration + test_duration))
         echo ""
         echo "========================================="
         if [[ -n "$PACKAGE_FLAGS" ]]; then
@@ -316,12 +309,87 @@ runs:
           echo "DAG scope:                     full workspace (${TOTAL_CRATES} crates)"
         fi
         echo "All targets build:             ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))"
-        echo "Tests compile:                 ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))"
-        echo "Tests execute:                 ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))"
+        echo "Tests compile and archive:     ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))"
         echo "-----------------------------------------"
         echo "Total build:                   ${build_duration}s ($(date -ud @${build_duration} +'%M:%S'))"
-        echo "Total time:                    ${total_duration}s ($(date -ud @${total_duration} +'%M:%S'))"
         echo "========================================="
+        du -h "$ARTIFACT_DIR"/*
+      shell: bash
+
+    - name: Upload archived Rust tests
+      if: inputs.task == 'test-build'
+      uses: actions/upload-artifact@v7
+      with:
+        name: rust-nextest-archive-${{ github.run_id }}-${{ github.run_attempt }}
+        path: ${{ runner.temp }}/rust-nextest-archive
+        compression-level: 0
+        if-no-files-found: error
+        retention-days: 1
+
+    - name: Download archived Rust tests
+      if: startsWith(inputs.task, 'test-run-')
+      uses: actions/download-artifact@v8
+      with:
+        name: rust-nextest-archive-${{ github.run_id }}-${{ github.run_attempt }}
+        path: ${{ runner.temp }}/rust-nextest-archive
+
+    - name: Run archived tests with coverage
+      if: startsWith(inputs.task, 'test-run-')
+      run: |
+        TASK="${{ inputs.task }}"
+        if [[ ! "$TASK" =~ ^test-run-([0-9]+)$ ]]; then
+          echo "::error::Invalid archived test task: $TASK"
+          exit 1
+        fi
+        PARTITION_INDEX="${BASH_REMATCH[1]}"
+        ARTIFACT_DIR="${RUNNER_TEMP}/rust-nextest-archive"
+        NEXTEST_FILTER=$(cat "$ARTIFACT_DIR/nextest-filter.txt")
+
+        mkdir -p target/debug
+        tar -C target/debug -xf "$ARTIFACT_DIR/connector-plugins.tar"
+        tar -C target/debug -xzf "$ARTIFACT_DIR/runtime-binaries.tar.gz"
+
+        RUNTIME_BINARIES=(
+          iggy
+          iggy-server
+          iggy-server-ng
+          iggy-connectors
+          iggy-mcp
+          iggy-bench
+        )
+        CARGO_BIN_ENV=()
+        for binary in "${RUNTIME_BINARIES[@]}"; do
+          binary_path="${GITHUB_WORKSPACE}/target/debug/${binary}"
+          if [[ ! -x "$binary_path" ]]; then
+            echo "::error::Required test runtime binary not found after extraction: ${binary_path}"
+            exit 1
+          fi
+          CARGO_BIN_ENV+=("CARGO_BIN_EXE_${binary}=${binary_path}")
+        done
+
+        source <(cargo llvm-cov show-env --sh)
+
+        if [[ "$RUNNER_OS" == "Linux" ]]; then
+          sudo sysctl -w vm.max_map_count=2000000 || true
+          eval "$(dbus-launch --sh-syntax)"
+          export DBUS_SESSION_BUS_ADDRESS
+          eval "$(echo -n "test" | gnome-keyring-daemon --unlock --components=secrets)"
+          echo -n "warmup" | secret-tool store --label="ci-warmup" ci-test warmup
+        fi
+
+        test_start=$(date +%s)
+        env "${CARGO_BIN_ENV[@]}" cargo nextest run \
+          --archive-file "$ARTIFACT_DIR/rust-tests.tar.zst" \
+          --extract-to "$GITHUB_WORKSPACE" \
+          --extract-overwrite \
+          --workspace-remap "$GITHUB_WORKSPACE" \
+          --no-fail-fast \
+          --profile ci \
+          --partition "hash:${PARTITION_INDEX}/2" \
+          -E "$NEXTEST_FILTER"
+        test_end=$(date +%s)
+        test_duration=$((test_end - test_start))
+        echo "::notice::Partition ${PARTITION_INDEX}/2 executed in ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))"
 
         cargo llvm-cov report --codecov --output-path codecov.json
         echo "Coverage report generated: codecov.json"

.github/actions/utils/setup-rust-with-cache/action.yml

@@ -19,6 +19,14 @@ name: setup-rust-with-cache
 description: Setup Rust toolchain with Swatinem/rust-cache
 
 inputs:
+  install-nextest:
+    description: "Whether to install cargo-nextest"
+    required: false
+    default: "false"
+  install-system-dependencies:
+    description: "Whether to install system packages required by Rust builds"
+    required: false
+    default: "true"
   read-cache:
     description: "Whether to read from cache"
     required: false
@@ -55,14 +63,14 @@ runs:
         aggressive: ${{ inputs.free-disk-space-aggressive }}
 
     - name: Install system dependencies (Linux)
-      if: runner.os == 'Linux'
+      if: runner.os == 'Linux' && inputs.install-system-dependencies == 'true'
       run: |
         sudo apt-get update
         sudo apt-get install -y libhwloc-dev pkg-config libudev-dev
       shell: bash
 
     - name: Install system dependencies (macOS)
-      if: runner.os == 'macOS'
+      if: runner.os == 'macOS' && inputs.install-system-dependencies == 'true'
       run: |
         # Pin version of hwloc to 2.12.2_1
         # brew extract doesn't have this version, so we fetch the formula directly
@@ -117,7 +125,7 @@ runs:
       shell: bash
 
     - name: Install cargo-nextest
-      if: runner.os == 'Linux'
+      if: runner.os == 'Linux' && inputs.install-nextest == 'true'
       run: |
         if command -v cargo-nextest &> /dev/null; then
           echo "cargo-nextest already installed"

.github/config/components.yml

@@ -174,8 +174,7 @@ components:
       - "sort"
       - "doctest"
       - "machete"
-      - "test-1"
-      - "test-2"
+      - "test"
       - "compat"
       - "build-aarch64-gnu"
       - "build-aarch64-musl"