test: reduce race UT critical path

.github/workflows/entrypoint.yaml

@@ -75,7 +75,10 @@ jobs:
     if: ${{ needs.check-pr-valid.outputs.pr_valid == 'true' && github.base_ref != '3.0-dev' }}
     uses: matrixorigin/CI/.github/workflows/ci.yaml@main
     with:
-      ut_parallel: 8
+      # Leave one runner CPU for race-detector/native work inside a package.
+      # Eight package slots made CPU-heavy HNSW tests the light-stage straggler.
+      ut_parallel: 7
+      ut_sharded: true
     secrets: inherit
 
   matrixone-ut-coverage:

Makefile

@@ -406,16 +406,19 @@ ut: $(UT_PREREQUISITES)
 ifeq ($(UNAME_S),darwin)
 	@cd optools && ./run_ut.sh UT $(SKIP_TEST)
 else
-	# The race suite is split into light, exclusive, heavy, and plan shards.
-	# Keep the outer budget above the per-package timeout so an expanded main
-	# branch cannot be killed while later shards are still making progress.
+	# The race suite is internally partitioned into light/HNSW, exclusive issues,
+	# embedded-cluster, heavy/engine, and plan stages. Keep the outer budget above
+	# the per-package timeout so an expanded main branch cannot be killed while a
+	# selected stage is still making progress.
 	@cd optools && timeout 90m ./run_ut.sh UT $(SKIP_TEST)
 endif
 
 ###############################################################################
 # bvt and unit test
 ###############################################################################
 UT_PARALLEL ?= 1
+UT_SHARD ?= all
+export UT_SHARD
 # Native compilation runs before Go tests, so it can use an explicit UT CPU
 # budget without increasing peak race-test memory. With the default UT value,
 # omit -j and preserve recursive make's jobserver contract: a plain make stays

optools/run_ut.sh

@@ -41,6 +41,7 @@ BUILD_WKSP=$(dirname "$PWD") && cd $BUILD_WKSP
 LOG="$G_TS-$TEST_TYPE.log"
 UT_TIMEOUT=${UT_TIMEOUT:-"15"}
 UT_PARALLEL=${UT_PARALLEL:-"1"}
+UT_SHARD=${UT_SHARD:-"all"}
 HEAVY_RACE_PARALLEL=${HEAVY_RACE_PARALLEL:-"3"}
 PLAN_RACE_SHARDS=${PLAN_RACE_SHARDS:-"8"}
 # Two shards cut the measured engine/test race runtime roughly in half while
@@ -54,6 +55,7 @@ CODE_COVERAGE="$G_WKSP/$G_TS-UT-Coverage.html"
 RAW_COVERAGE="coverage.out"
 IS_BUILD_FAIL=""
 UT_TEST_STATUS=0
+UT_SHARD_ROUTING_ERROR=0
 PLAN_RACE_TEST_BINARY=""
 ENGINE_RACE_TEST_BINARY=""
 ENGINE_RACE_JOB_PID=""
@@ -490,14 +492,48 @@ function run_tests(){
     echo "#  COVERAGE REPORT: $CODE_COVERAGE"
     echo "#  UT TIMEOUT:      $UT_TIMEOUT"
     echo "#  UT PARALLEL:     $UT_PARALLEL"
+    echo "#  UT SHARD:        $UT_SHARD"
     echo "#  CLUSTER ADMISSION: process lifecycle"
     echo "#  HEAVY RACE UT:   $HEAVY_RACE_PARALLEL total package slots"
     horiz_rule
 
+    if ! list_ut_shard_stages "${UT_SHARD}" >/dev/null; then
+        logger "ERR" "UT_SHARD must be all, light, issues, embedded, or heavy-plan; got '${UT_SHARD}'"
+        UT_TEST_STATUS=1
+        return 0
+    fi
+    if ! validate_complete_partition \
+        "UT stage" \
+        "$(list_ut_shard_stages all)" \
+        "$(list_ut_shard_stages light)" \
+        "$(list_ut_shard_stages issues)" \
+        "$(list_ut_shard_stages embedded)" \
+        "$(list_ut_shard_stages heavy-plan)"; then
+        logger "ERR" "Race-UT shard stages are not a complete disjoint partition"
+        UT_TEST_STATUS=1
+        return 0
+    fi
+    if [[ "${SKIP_TESTS}" == "race" && "${UT_SHARD}" != "all" ]]; then
+        logger "ERR" "split UT shards require race mode; got SKIP_TESTS=race with UT_SHARD=${UT_SHARD}"
+        UT_TEST_STATUS=1
+        return 0
+    fi
+
     logger "INF" "Clean go test cache"
     go clean -testcache
 
-    local test_scope=$(go list ${GO_MODULE_MODE} ./... | grep -v 'driver/aoe' | grep -v 'engine/aoe' | grep -v 'pkg/catalog')
+    local test_scope
+    if ! test_scope=$(go list ${GO_MODULE_MODE} ./...); then
+        logger "ERR" "Failed to resolve the complete UT package scope"
+        UT_TEST_STATUS=1
+        return 0
+    fi
+    test_scope=$(printf '%s\n' "${test_scope}" | grep -v 'driver/aoe' | grep -v 'engine/aoe' | grep -v 'pkg/catalog')
+    if [[ -z "${test_scope}" ]]; then
+        logger "ERR" "The complete UT package scope is empty"
+        UT_TEST_STATUS=1
+        return 0
+    fi
     local leave_out=$(egrep -lr  --include="*.go" 'Code generated by protoc-gen-gogo. DO NOT EDIT.' ./pkg/* | sort -u | xargs basename -a)
     logger "INF" "Ingore code coverage $(echo ${leave_out[@]}|tr " " "|")"
     local cover_profile='profile.raw'
@@ -526,6 +562,8 @@ function run_tests(){
         logger "INF" "Run UT with race check"
         local plan_package
         local engine_package
+        local hnsw_package
+        local engine_test_scope
         local serial_test_scope
         local cluster_test_scope
         local resource_heavy_test_scope
@@ -534,6 +572,7 @@ function run_tests(){
         local cluster_package_parallel=2
         local package_status=0
         local light_status=0
+        local hnsw_status=0
         local serial_status=0
         local cluster_status=0
         local resource_heavy_status=0
@@ -563,16 +602,20 @@ function run_tests(){
             UT_TEST_STATUS=1
             return 0
         fi
+        if ! hnsw_package=$(go list ${GO_MODULE_MODE} ./pkg/vectorindex/hnsw); then
+            logger "ERR" "Failed to resolve ./pkg/vectorindex/hnsw"
+            UT_TEST_STATUS=1
+            return 0
+        fi
 
-        # These packages need exclusive runner access. NewTestService callers
-        # bind fixed ports, while the issues packages intentionally keep embedded
-        # clusters alive for most of their test processes.
+        # The main issues package keeps its shared base cluster alive for most
+        # of the test process, so it retains an exclusive runner. The isolated
+        # issues package has no shared base and safely belongs to the embedded
+        # group: runner-wide admission still serializes each complete cluster
+        # lifecycle. Former logservice/TAE members allocate independent ports
+        # with collision retry and belong in the normal parallel scope.
         if ! serial_test_scope=$(go list ${GO_MODULE_MODE} \
-            ./pkg/logservice \
-            ./pkg/vm/engine/tae/logstore \
-            ./pkg/vm/engine/tae/logstore/driver/logservicedriver \
-            ./pkg/tests/issues \
-            ./pkg/tests/issues/isolated); then
+            ./pkg/tests/issues); then
             logger "ERR" "Failed to resolve serial race-test packages"
             UT_TEST_STATUS=1
             return 0
@@ -593,6 +636,7 @@ function run_tests(){
         cluster_test_scope=$(remove_packages_from_scope \
             "${cluster_test_scope}" \
             "${plan_package}" \
+            "${hnsw_package}" \
             ${serial_test_scope})
 
         # Dependency-based group precedence remains authoritative. If this
@@ -601,7 +645,10 @@ function run_tests(){
         if printf '%s\n%s\n' "${serial_test_scope}" "${cluster_test_scope}" |
             grep -Fxq "${engine_package}"; then
             shard_engine=0
+            engine_test_scope=""
             logger "INF" "Keep ${engine_package} in its higher-precedence race-test group"
+        else
+            engine_test_scope="${engine_package}"
         fi
 
         if ! resource_heavy_test_scope=$(go list ${GO_MODULE_MODE} \
@@ -617,46 +664,76 @@ function run_tests(){
             "${resource_heavy_test_scope}" \
             "${plan_package}" \
             "${engine_package}" \
+            "${hnsw_package}" \
             ${serial_test_scope} \
             ${cluster_test_scope})
 
         light_test_scope=$(remove_packages_from_scope \
             "${test_scope}" \
             "${plan_package}" \
             "${engine_package}" \
+            "${hnsw_package}" \
             ${serial_test_scope} \
             ${cluster_test_scope} \
             ${resource_heavy_test_scope})
 
-        if [[ -n "${light_test_scope}" ]]; then
+        if ! validate_complete_partition \
+            "UT package" \
+            "${test_scope}" \
+            "${light_test_scope}" \
+            "${hnsw_package}" \
+            "${serial_test_scope}" \
+            "${cluster_test_scope}" \
+            "${resource_heavy_test_scope}" \
+            "${engine_test_scope}" \
+            "${plan_package}"; then
+            logger "ERR" "Race-UT package groups are not a complete disjoint partition"
+            UT_TEST_STATUS=1
+            return 0
+        fi
+
+        : > "${UT_REPORT}"
+        if should_run_ut_stage light && [[ -n "${light_test_scope}" ]]; then
             logger "INF" "Run light race-test packages with parallelism ${UT_PARALLEL}"
-            LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p ${UT_PARALLEL} -timeout "${UT_TIMEOUT}m" -race $light_test_scope > $UT_REPORT
+            LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p ${UT_PARALLEL} -timeout "${UT_TIMEOUT}m" -race $light_test_scope >> $UT_REPORT
             light_status=$?
-        else
-            : > "${UT_REPORT}"
         fi
 
-        logger "INF" "Run exclusive race-test packages serially"
-        for package in ${serial_test_scope}; do
-            logger "INF" "Run exclusive race-test package ${package}"
-            LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p 1 -timeout "${UT_TIMEOUT}m" -race "${package}" >> $UT_REPORT
-            package_status=$?
-            if (( package_status != 0 )); then
-                serial_status=1
-                logger "ERR" "Exclusive race-test package ${package} failed with status ${package_status}"
-            fi
-        done
+        # HNSW owns native worker pools inside its test binary. Running it as
+        # one package slot after the normal light wave preserves its full-batch
+        # and repeated-lifecycle targets without letting package concurrency
+        # turn CPU scheduling delay into a multi-minute light-stage straggler.
+        if should_run_ut_stage hnsw; then
+            logger "INF" "Run HNSW race-test package with exclusive runner CPU"
+            LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p 1 -timeout "${UT_TIMEOUT}m" -race "${hnsw_package}" >> $UT_REPORT
+            hnsw_status=$?
+        fi
+
+        if should_run_ut_stage serial; then
+            logger "INF" "Run exclusive race-test packages serially"
+            for package in ${serial_test_scope}; do
+                logger "INF" "Run exclusive race-test package ${package}"
+                LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p 1 -timeout "${UT_TIMEOUT}m" -race "${package}" >> $UT_REPORT
+                package_status=$?
+                if (( package_status != 0 )); then
+                    serial_status=1
+                    logger "ERR" "Exclusive race-test package ${package} failed with status ${package_status}"
+                fi
+            done
+        fi
 
         # These packages link embedded clusters with substantial race-detector
         # memory. The runner-wide file-lock admission keeps complete cluster
         # lifecycles serialized across test binaries. Allow one additional
         # package process to overlap linking, setup, and non-cluster work without
         # returning to the six-way contention that starved HAKeeper.
-        logger "INF" "Run embedded-cluster race-test packages with package parallelism ${cluster_package_parallel} and serialized cluster lifecycle admission"
-        LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p "${cluster_package_parallel}" -timeout "${UT_TIMEOUT}m" -race $cluster_test_scope >> $UT_REPORT
-        cluster_status=$?
+        if should_run_ut_stage embedded; then
+            logger "INF" "Run embedded-cluster race-test packages with package parallelism ${cluster_package_parallel} and serialized cluster lifecycle admission"
+            LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p "${cluster_package_parallel}" -timeout "${UT_TIMEOUT}m" -race $cluster_test_scope >> $UT_REPORT
+            cluster_status=$?
+        fi
 
-        if (( shard_engine == 1 )); then
+        if should_run_ut_stage heavy && (( shard_engine == 1 )); then
             # engine/test is dominated by serial fixture lifecycles inside one
             # process. Build it once and split every discovered top-level test
             # across fresh race processes. The effective shard count and the
@@ -679,42 +756,46 @@ function run_tests(){
             else
                 resource_heavy_parallel=${HEAVY_RACE_PARALLEL}
             fi
-        else
+        elif should_run_ut_stage heavy; then
             resource_heavy_parallel=${HEAVY_RACE_PARALLEL}
         fi
 
-        logger "INF" "Run remaining resource-heavy race-test packages with parallelism ${resource_heavy_parallel}"
-        LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p ${resource_heavy_parallel} -timeout "${UT_TIMEOUT}m" -race $resource_heavy_test_scope >> $UT_REPORT
-        resource_heavy_status=$?
-
-        if (( shard_engine == 1 )); then
-            if [[ -n "${ENGINE_RACE_JOB_PID}" ]]; then
-                wait "${ENGINE_RACE_JOB_PID}"
-                engine_status=$?
-                ENGINE_RACE_JOB_PID=""
-            else
-                # Keep the helper's process-group TERM trap scoped to a
-                # subshell even when a low budget requires sequential waves.
-                run_engine_race_shards "${engine_package}" "${engine_race_parallel}" &
-                ENGINE_RACE_JOB_PID=$!
-                wait "${ENGINE_RACE_JOB_PID}"
-                engine_status=$?
-                ENGINE_RACE_JOB_PID=""
+        if should_run_ut_stage heavy; then
+            logger "INF" "Run remaining resource-heavy race-test packages with parallelism ${resource_heavy_parallel}"
+            LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p ${resource_heavy_parallel} -timeout "${UT_TIMEOUT}m" -race $resource_heavy_test_scope >> $UT_REPORT
+            resource_heavy_status=$?
+
+            if (( shard_engine == 1 )); then
+                if [[ -n "${ENGINE_RACE_JOB_PID}" ]]; then
+                    wait "${ENGINE_RACE_JOB_PID}"
+                    engine_status=$?
+                    ENGINE_RACE_JOB_PID=""
+                else
+                    # Keep the helper's process-group TERM trap scoped to a
+                    # subshell even when a low budget requires sequential waves.
+                    run_engine_race_shards "${engine_package}" "${engine_race_parallel}" &
+                    ENGINE_RACE_JOB_PID=$!
+                    wait "${ENGINE_RACE_JOB_PID}"
+                    engine_status=$?
+                    ENGINE_RACE_JOB_PID=""
+                fi
+                if [[ -s "${ENGINE_RACE_REPORT}" ]]; then
+                    cat "${ENGINE_RACE_REPORT}" >> "${UT_REPORT}"
+                fi
+                rm -f "${ENGINE_RACE_TEST_BINARY}" "${ENGINE_RACE_REPORT}" "${ENGINE_RACE_REPORT}".*
+                ENGINE_RACE_TEST_BINARY=""
+                ENGINE_RACE_REPORT=""
             fi
-            if [[ -s "${ENGINE_RACE_REPORT}" ]]; then
-                cat "${ENGINE_RACE_REPORT}" >> "${UT_REPORT}"
-            fi
-            rm -f "${ENGINE_RACE_TEST_BINARY}" "${ENGINE_RACE_REPORT}" "${ENGINE_RACE_REPORT}".*
-            ENGINE_RACE_TEST_BINARY=""
-            ENGINE_RACE_REPORT=""
-        fi
 
-        report_cgroup_memory_usage "Resource-heavy UT"
+            report_cgroup_memory_usage "Resource-heavy UT"
+        fi
 
-        run_plan_race_shards "${plan_package}"
-        plan_status=$?
+        if should_run_ut_stage plan; then
+            run_plan_race_shards "${plan_package}"
+            plan_status=$?
+        fi
 
-        if (( light_status != 0 || serial_status != 0 || cluster_status != 0 || resource_heavy_status != 0 || engine_status != 0 || plan_status != 0 )); then
+        if (( UT_SHARD_ROUTING_ERROR != 0 || light_status != 0 || hnsw_status != 0 || serial_status != 0 || cluster_status != 0 || resource_heavy_status != 0 || engine_status != 0 || plan_status != 0 )); then
             UT_TEST_STATUS=1
         fi
     fi