diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index 536e3056e0..1dec5a41c7 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -20,7 +20,7 @@ description: Rust pre-merge testing and linting github iggy actions inputs: task: - description: "Task to run (check, check-msrv, fmt, clippy, sort, machete, doctest, verify-publish, test-1, test-2, compat, miri)" + description: "Task to run (check, check-msrv, fmt, clippy, sort, machete, doctest, verify-publish, test-build, test-run-1, test-run-2, compat, miri)" required: true component: description: "Component name (for context)" @@ -33,6 +33,12 @@ runs: - name: Setup Rust with cache uses: ./.github/actions/utils/setup-rust-with-cache with: + # Only test jobs execute nextest. Avoid downloading it in lint, metadata, + # and cross-build jobs. + install-nextest: ${{ inputs.task == 'test-build' || startsWith(inputs.task, 'test-run-') }} + # These tasks do not compile workspace crates and need no native + # libraries. All compiling tasks keep the existing dependency setup. + install-system-dependencies: ${{ inputs.task != 'fmt' && inputs.task != 'sort' && inputs.task != 'machete' }} # Miri builds against a nightly toolchain with a separate `target/miri` # subtree; isolate its cache from the stable `dev` namespace so the # two don't evict each other. @@ -45,7 +51,7 @@ runs: # fmt/sort/machete never build into target/, so the multi-GB cache # restore is pure overhead. Compiling legs (check/clippy/doctest/test-*) # keep it; a cold cache there recompiles the whole dep tree. - read-cache: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'machete') && 'false' || 'true' }} + read-cache: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'machete' || startsWith(inputs.task, 'test-run-')) && 'false' || 'true' }} # Light legs fit in the runner's ~89 GiB default headroom; skip the # ~20-45s reclaim. Disk-heavy legs (coverage build + testcontainers # images on test-*, cross-builds, miri, verify-publish) keep it. @@ -79,18 +85,18 @@ runs: # Safety: cargo check/clippy run on the full workspace separately, catching all # compilation errors. This only scopes test BUILD and EXECUTION. - name: Fetch base branch for DAG analysis - if: startsWith(inputs.task, 'test-') + if: inputs.task == 'test-build' run: git fetch origin master --depth=1 2>/dev/null || true shell: bash - name: Install cargo-rail - if: startsWith(inputs.task, 'test-') + if: inputs.task == 'test-build' uses: taiki-e/install-action@v2 with: tool: cargo-rail - name: Compute affected crates (cargo-rail) - if: startsWith(inputs.task, 'test-') + if: inputs.task == 'test-build' run: | METADATA_JSON=$(cargo metadata --format-version 1 --no-deps 2>/dev/null || echo "{}") TOTAL_CRATES=$(echo "$METADATA_JSON" | jq '.workspace_members | length' 2>/dev/null || echo "?") @@ -110,8 +116,6 @@ runs: CRATE_COUNT=$(echo "$CRATES" | wc -l) # Build nextest filter expression for cargo nextest run (affected crates only) echo "$CRATES" | sed 's/^/package(/; s/$/)/' | paste -sd '|' | sed 's/|/ | /g' > /tmp/nextest-filter.txt - # Save affected-only -p flags for cargo test fallback (no nextest filter) - echo "$CRATES" | sed 's/^/-p /' | tr '\n' ' ' > /tmp/test-packages.txt # Build -p flags: affected crates + packages with bin/cdylib targets. # Binary packages: nextest sets CARGO_BIN_EXE_ from compiled # artifacts; cdylib packages: connector plugins loaded via dlopen. @@ -187,34 +191,23 @@ runs: shell: bash - name: Install dependencies for Rust tests - if: startsWith(inputs.task, 'test-') && runner.os == 'Linux' + if: startsWith(inputs.task, 'test-run-') && runner.os == 'Linux' run: | sudo apt-get install --yes musl-tools gnome-keyring keyutils dbus-x11 libsecret-tools rm -f $HOME/.local/share/keyrings/* shell: bash - name: Install cargo-llvm-cov - if: startsWith(inputs.task, 'test-') + if: inputs.task == 'test-build' || startsWith(inputs.task, 'test-run-') uses: taiki-e/install-action@v2 with: tool: cargo-llvm-cov - - name: Build and test with coverage - if: startsWith(inputs.task, 'test-') + - name: Build and archive tests with coverage + if: inputs.task == 'test-build' run: | - # Parse partition index from task name (test-1 -> hash:1/2, test-2 -> hash:2/2) - TASK="${{ inputs.task }}" - PARTITION_FLAG="" - if [[ "$TASK" =~ ^test-([0-9]+)$ ]]; then - PARTITION_INDEX="${BASH_REMATCH[1]}" - PARTITION_FLAG="--partition hash:${PARTITION_INDEX}/2" - echo "::notice::Running test partition ${PARTITION_INDEX}/2" - fi - - # Read DAG-based affected crate filter (computed in earlier step) NEXTEST_FILTER="" PACKAGE_FLAGS="" - TEST_PACKAGE_FLAGS="" TOTAL_CRATES="?" if [[ -f /tmp/nextest-filter.txt ]]; then NEXTEST_FILTER=$(cat /tmp/nextest-filter.txt) @@ -222,9 +215,6 @@ runs: if [[ -f /tmp/packages.txt ]]; then PACKAGE_FLAGS=$(cat /tmp/packages.txt) fi - if [[ -f /tmp/test-packages.txt ]]; then - TEST_PACKAGE_FLAGS=$(cat /tmp/test-packages.txt) - fi if [[ -f /tmp/total-crates.txt ]]; then TOTAL_CRATES=$(cat /tmp/total-crates.txt) fi @@ -237,17 +227,7 @@ runs: echo "::notice::Full workspace build (no DAG filter available)" fi - source <(cargo llvm-cov show-env --export-prefix) - - # Doris 4.0.3's start_be.sh hard-`exit 1`s unless vm.max_map_count >= 2000000. - # This kernel param can only be set on the host (no container can raise it), - # so we raise it here. Everything else about booting Doris — image, heap/mem - # caps, port mappings, BE-alive wait — lives in the testcontainers fixture - # at core/integration/tests/connectors/fixtures/doris/container.rs, so - # `cargo test` and CI follow exactly the same path. - if [[ "$RUNNER_OS" == "Linux" ]]; then - sudo sysctl -w vm.max_map_count=2000000 || true - fi + source <(cargo llvm-cov show-env --sh) bins_start=$(date +%s) if [[ -n "$PACKAGE_FLAGS" ]]; then @@ -259,52 +239,65 @@ runs: bins_duration=$((bins_end - bins_start)) echo "::notice::Binaries and libraries built in ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))" + ARTIFACT_DIR="${RUNNER_TEMP}/rust-nextest-archive" + mkdir -p "$ARTIFACT_DIR" + compile_start=$(date +%s) if [[ -n "$PACKAGE_FLAGS" ]]; then - cargo test --locked --no-run $PACKAGE_FLAGS + cargo nextest archive --locked $PACKAGE_FLAGS \ + --archive-file "$ARTIFACT_DIR/rust-tests.tar.zst" else - cargo test --locked --no-run + cargo nextest archive --locked \ + --archive-file "$ARTIFACT_DIR/rust-tests.tar.zst" fi compile_end=$(date +%s) compile_duration=$((compile_end - compile_start)) - echo "::notice::Tests compiled in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" + echo "::notice::Tests compiled and archived in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" - # Start D-Bus and unlock keyring right before test execution to avoid - # gnome-keyring auto-locking the collection during the build phase. - # Previously this ran before `cargo build`, leaving a 7+ minute idle - # window that triggered org.freedesktop.Secret.Error.IsLocked ~10% of runs. - if [[ "$RUNNER_OS" == "Linux" ]]; then - eval $(dbus-launch --sh-syntax) - export DBUS_SESSION_BUS_ADDRESS - eval $(echo -n "test" | gnome-keyring-daemon --unlock --components=secrets) - echo -n "warmup" | secret-tool store --label="ci-warmup" ci-test warmup + if [[ -n "$NEXTEST_FILTER" ]]; then + printf '%s\n' "$NEXTEST_FILTER" > "$ARTIFACT_DIR/nextest-filter.txt" + else + printf '%s\n' 'all()' > "$ARTIFACT_DIR/nextest-filter.txt" fi - test_start=$(date +%s) - if command -v cargo-nextest &> /dev/null; then - if [[ -n "$NEXTEST_FILTER" ]]; then - cargo nextest run --locked --no-fail-fast --profile ci $PARTITION_FLAG $PACKAGE_FLAGS -E "$NEXTEST_FILTER" - else - cargo nextest run --locked --no-fail-fast --profile ci $PARTITION_FLAG - fi + PLUGIN_FILES=() + while IFS= read -r -d '' plugin_file; do + PLUGIN_FILES+=("${plugin_file##*/}") + done < <( + find target/debug -maxdepth 1 -type f \ + -name 'libiggy_connector_*.so' -print0 + ) + if (( ${#PLUGIN_FILES[@]} > 0 )); then + tar -C target/debug -cf "$ARTIFACT_DIR/connector-plugins.tar" \ + "${PLUGIN_FILES[@]}" else - if [[ -n "$PARTITION_FLAG" ]]; then - echo "::error::cargo-nextest not found, falling back to cargo test without partitioning (all tests will run on every partition)" - fi - # Use TEST_PACKAGE_FLAGS (affected crates only), not PACKAGE_FLAGS - # (which includes binary packages whose tests should NOT run). - if [[ -n "$TEST_PACKAGE_FLAGS" ]]; then - cargo test --locked --no-fail-fast $TEST_PACKAGE_FLAGS - else - cargo test --locked --no-fail-fast - fi + echo "::error::No connector plugin libraries found in target/debug" + exit 1 fi - test_end=$(date +%s) - test_duration=$((test_end - test_start)) - echo "::notice::Tests executed in ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))" + + # The integration package launches binaries owned by other workspace + # packages through assert_cmd::cargo_bin. Nextest only exposes + # CARGO_BIN_EXE_* for binaries Cargo associates with the current + # integration-test package. Preserve these cross-package executables + # explicitly so test-run jobs can provide the expected paths. + RUNTIME_BINARIES=( + iggy + iggy-server + iggy-server-ng + iggy-connectors + iggy-mcp + iggy-bench + ) + for binary in "${RUNTIME_BINARIES[@]}"; do + if [[ ! -x "target/debug/${binary}" ]]; then + echo "::error::Required test runtime binary not found: target/debug/${binary}" + exit 1 + fi + done + tar -C target/debug -czf "$ARTIFACT_DIR/runtime-binaries.tar.gz" \ + "${RUNTIME_BINARIES[@]}" build_duration=$((bins_duration + compile_duration)) - total_duration=$((build_duration + test_duration)) echo "" echo "=========================================" if [[ -n "$PACKAGE_FLAGS" ]]; then @@ -316,12 +309,87 @@ runs: echo "DAG scope: full workspace (${TOTAL_CRATES} crates)" fi echo "All targets build: ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))" - echo "Tests compile: ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" - echo "Tests execute: ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))" + echo "Tests compile and archive: ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" echo "-----------------------------------------" echo "Total build: ${build_duration}s ($(date -ud @${build_duration} +'%M:%S'))" - echo "Total time: ${total_duration}s ($(date -ud @${total_duration} +'%M:%S'))" echo "=========================================" + du -h "$ARTIFACT_DIR"/* + shell: bash + + - name: Upload archived Rust tests + if: inputs.task == 'test-build' + uses: actions/upload-artifact@v7 + with: + name: rust-nextest-archive-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/rust-nextest-archive + compression-level: 0 + if-no-files-found: error + retention-days: 1 + + - name: Download archived Rust tests + if: startsWith(inputs.task, 'test-run-') + uses: actions/download-artifact@v8 + with: + name: rust-nextest-archive-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/rust-nextest-archive + + - name: Run archived tests with coverage + if: startsWith(inputs.task, 'test-run-') + run: | + TASK="${{ inputs.task }}" + if [[ ! "$TASK" =~ ^test-run-([0-9]+)$ ]]; then + echo "::error::Invalid archived test task: $TASK" + exit 1 + fi + PARTITION_INDEX="${BASH_REMATCH[1]}" + ARTIFACT_DIR="${RUNNER_TEMP}/rust-nextest-archive" + NEXTEST_FILTER=$(cat "$ARTIFACT_DIR/nextest-filter.txt") + + mkdir -p target/debug + tar -C target/debug -xf "$ARTIFACT_DIR/connector-plugins.tar" + tar -C target/debug -xzf "$ARTIFACT_DIR/runtime-binaries.tar.gz" + + RUNTIME_BINARIES=( + iggy + iggy-server + iggy-server-ng + iggy-connectors + iggy-mcp + iggy-bench + ) + CARGO_BIN_ENV=() + for binary in "${RUNTIME_BINARIES[@]}"; do + binary_path="${GITHUB_WORKSPACE}/target/debug/${binary}" + if [[ ! -x "$binary_path" ]]; then + echo "::error::Required test runtime binary not found after extraction: ${binary_path}" + exit 1 + fi + CARGO_BIN_ENV+=("CARGO_BIN_EXE_${binary}=${binary_path}") + done + + source <(cargo llvm-cov show-env --sh) + + if [[ "$RUNNER_OS" == "Linux" ]]; then + sudo sysctl -w vm.max_map_count=2000000 || true + eval "$(dbus-launch --sh-syntax)" + export DBUS_SESSION_BUS_ADDRESS + eval "$(echo -n "test" | gnome-keyring-daemon --unlock --components=secrets)" + echo -n "warmup" | secret-tool store --label="ci-warmup" ci-test warmup + fi + + test_start=$(date +%s) + env "${CARGO_BIN_ENV[@]}" cargo nextest run \ + --archive-file "$ARTIFACT_DIR/rust-tests.tar.zst" \ + --extract-to "$GITHUB_WORKSPACE" \ + --extract-overwrite \ + --workspace-remap "$GITHUB_WORKSPACE" \ + --no-fail-fast \ + --profile ci \ + --partition "hash:${PARTITION_INDEX}/2" \ + -E "$NEXTEST_FILTER" + test_end=$(date +%s) + test_duration=$((test_end - test_start)) + echo "::notice::Partition ${PARTITION_INDEX}/2 executed in ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))" cargo llvm-cov report --codecov --output-path codecov.json echo "Coverage report generated: codecov.json" diff --git a/.github/actions/utils/setup-rust-with-cache/action.yml b/.github/actions/utils/setup-rust-with-cache/action.yml index e01ce81f8d..6c3bf16fb4 100644 --- a/.github/actions/utils/setup-rust-with-cache/action.yml +++ b/.github/actions/utils/setup-rust-with-cache/action.yml @@ -19,6 +19,14 @@ name: setup-rust-with-cache description: Setup Rust toolchain with Swatinem/rust-cache inputs: + install-nextest: + description: "Whether to install cargo-nextest" + required: false + default: "false" + install-system-dependencies: + description: "Whether to install system packages required by Rust builds" + required: false + default: "true" read-cache: description: "Whether to read from cache" required: false @@ -55,14 +63,14 @@ runs: aggressive: ${{ inputs.free-disk-space-aggressive }} - name: Install system dependencies (Linux) - if: runner.os == 'Linux' + if: runner.os == 'Linux' && inputs.install-system-dependencies == 'true' run: | sudo apt-get update sudo apt-get install -y libhwloc-dev pkg-config libudev-dev shell: bash - name: Install system dependencies (macOS) - if: runner.os == 'macOS' + if: runner.os == 'macOS' && inputs.install-system-dependencies == 'true' run: | # Pin version of hwloc to 2.12.2_1 # brew extract doesn't have this version, so we fetch the formula directly @@ -117,7 +125,7 @@ runs: shell: bash - name: Install cargo-nextest - if: runner.os == 'Linux' + if: runner.os == 'Linux' && inputs.install-nextest == 'true' run: | if command -v cargo-nextest &> /dev/null; then echo "cargo-nextest already installed" diff --git a/.github/config/components.yml b/.github/config/components.yml index 2d30025cb5..4f6abcedcd 100644 --- a/.github/config/components.yml +++ b/.github/config/components.yml @@ -174,8 +174,7 @@ components: - "sort" - "doctest" - "machete" - - "test-1" - - "test-2" + - "test" - "compat" - "build-aarch64-gnu" - "build-aarch64-musl" diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 3bfdb057d1..f758d01d6d 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -38,6 +38,7 @@ permissions: jobs: run: + if: ${{ !(startsWith(inputs.component, 'rust') && inputs.task == 'test') }} # Select runner based on build target runs-on: ${{ (inputs.task == 'build-aarch64-gnu' || inputs.task == 'build-aarch64-musl') && 'ubuntu-24.04-arm' || @@ -62,18 +63,6 @@ jobs: task: ${{ inputs.task }} component: ${{ inputs.component }} - - name: Upload coverage to Codecov - if: startsWith(inputs.component, 'rust') && startsWith(inputs.task, 'test-') - uses: codecov/codecov-action@v7.0.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: codecov.json - disable_search: true - flags: rust - fail_ci_if_error: false - verbose: true - override_pr: ${{ github.event.pull_request.number }} - # Python SDK - name: Set up Docker Buildx for Python if: inputs.component == 'sdk-python' && inputs.task == 'test' @@ -262,3 +251,58 @@ jobs: codecov.json if-no-files-found: ignore retention-days: 7 + + build-rust-tests: + if: ${{ startsWith(inputs.component, 'rust') && inputs.task == 'test' }} + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v7.0.0 + + - name: Build and archive Rust tests + uses: ./.github/actions/rust/pre-merge + with: + task: test-build + component: ${{ inputs.component }} + + run-rust-tests: + if: ${{ startsWith(inputs.component, 'rust') && inputs.task == 'test' }} + needs: build-rust-tests + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + partition: [1, 2] + steps: + - name: Checkout code + uses: actions/checkout@v7.0.0 + + - name: Run Rust test partition + uses: ./.github/actions/rust/pre-merge + with: + task: test-run-${{ matrix.partition }} + component: ${{ inputs.component }} + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v7.0.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: codecov.json + disable_search: true + flags: rust + fail_ci_if_error: false + verbose: true + override_pr: ${{ github.event.pull_request.number }} + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: rust-test-${{ matrix.partition }}-reports-${{ github.run_id }}-${{ github.run_attempt }} + path: | + target/llvm-cov/** + codecov.json + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml index af3516c85c..758445e68c 100644 --- a/.github/workflows/coverage-baseline.yml +++ b/.github/workflows/coverage-baseline.yml @@ -52,6 +52,7 @@ jobs: - name: Setup Rust with cache uses: ./.github/actions/utils/setup-rust-with-cache with: + install-nextest: "true" # Also warms the GitHub Actions build cache for subsequent PR builds save-cache: "true" # llvm-cov instrumentation roughly doubles object sizes; the light diff --git a/scripts/ci/sync-python-interpreter-version.sh b/scripts/ci/sync-python-interpreter-version.sh index bc6a2de293..4c71c3ef2f 100755 --- a/scripts/ci/sync-python-interpreter-version.sh +++ b/scripts/ci/sync-python-interpreter-version.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information