From 331ef14de6e318abbf2143479c6ceb5c6ca0c8df Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Fri, 28 Aug 2026 22:53:12 +0800 Subject: [PATCH 1/7] test: reduce race UT critical path --- .github/workflows/entrypoint.yaml | 4 ++- optools/run_ut.sh | 10 +++---- pkg/vectorindex/hnsw/search_test.go | 10 +++---- pkg/vectorindex/hnsw/sync_test.go | 44 ++++++++++++++++++++++------- 4 files changed, 46 insertions(+), 22 deletions(-) diff --git a/.github/workflows/entrypoint.yaml b/.github/workflows/entrypoint.yaml index 444bb082b4a73..5d1b547dae146 100644 --- a/.github/workflows/entrypoint.yaml +++ b/.github/workflows/entrypoint.yaml @@ -75,7 +75,9 @@ jobs: if: ${{ needs.check-pr-valid.outputs.pr_valid == 'true' && github.base_ref != '3.0-dev' }} uses: matrixorigin/CI/.github/workflows/ci.yaml@main with: - ut_parallel: 8 + # Leave one runner CPU for race-detector/native work inside a package. + # Eight package slots made CPU-heavy HNSW tests the light-stage straggler. + ut_parallel: 7 secrets: inherit matrixone-ut-coverage: diff --git a/optools/run_ut.sh b/optools/run_ut.sh index 4ec818c7bdeff..efcf0c3cf1250 100755 --- a/optools/run_ut.sh +++ b/optools/run_ut.sh @@ -564,13 +564,11 @@ function run_tests(){ return 0 fi - # These packages need exclusive runner access. NewTestService callers - # bind fixed ports, while the issues packages intentionally keep embedded - # clusters alive for most of their test processes. + # The issues packages intentionally keep embedded clusters alive for + # most of their test processes, so they retain exclusive runner access. + # The former logservice/TAE members of this group now allocate independent + # ports with collision retry and belong in the normal parallel scope. if ! serial_test_scope=$(go list ${GO_MODULE_MODE} \ - ./pkg/logservice \ - ./pkg/vm/engine/tae/logstore \ - ./pkg/vm/engine/tae/logstore/driver/logservicedriver \ ./pkg/tests/issues \ ./pkg/tests/issues/isolated); then logger "ERR" "Failed to resolve serial race-test packages" diff --git a/pkg/vectorindex/hnsw/search_test.go b/pkg/vectorindex/hnsw/search_test.go index 516ec3ff7c42b..04b817526c4b7 100644 --- a/pkg/vectorindex/hnsw/search_test.go +++ b/pkg/vectorindex/hnsw/search_test.go @@ -314,11 +314,11 @@ func makeIndexBatch(proc *process.Process) *batch.Batch { } func TestFallocate(t *testing.T) { - - f, err := os.Create("apple") - require.Nil(t, err) - fallocate.Fallocate(f, 0, 10000) - f.Close() + f, err := os.CreateTemp(t.TempDir(), "fallocate-") + require.NoError(t, err) + t.Cleanup(func() { _ = f.Close() }) + require.NoError(t, fallocate.Fallocate(f, 0, 10000)) + require.NoError(t, f.Close()) } func makeMetaBatch2Files(proc *process.Process) *batch.Batch { diff --git a/pkg/vectorindex/hnsw/sync_test.go b/pkg/vectorindex/hnsw/sync_test.go index 512dc079b4cc1..67f2d0654e8dd 100644 --- a/pkg/vectorindex/hnsw/sync_test.go +++ b/pkg/vectorindex/hnsw/sync_test.go @@ -691,25 +691,41 @@ func runSyncContinuousUpdateInsertShuffle2FilesWithSmallCap[T types.RealNumbers] } }) + oldRunSQL := runSql + oldRunSQLStreaming := runSql_streaming + oldRunTxn := runTxn + t.Cleanup(func() { + runSql = oldRunSQL + runSql_streaming = oldRunSQLStreaming + runTxn = oldRunTxn + }) runSql = mock_runSql_2files runSql_streaming = mock_runSql_streaming_2files runTxn = mock_runTxn indexes := mockMoIndexes() - cdc := vectorindex.VectorIndexCdc[T]{Data: make([]vectorindex.VectorIndexCdcEntry[T], 0, 100)} - - key := int64(0) + // The fixture has two existing files containing keys 0..199. Exercise one + // update in each file plus eleven inserts: at capacity ten, the inserts must + // roll over into two new models. Thirteen entries also keep all eight build + // workers active. Preserve the original ten Update cycles as repeated + // lifecycle/stability coverage, while removing entries that only duplicated + // work inside each cycle and became prohibitively expensive under -race. + keys := []int64{0, 100} + for key := int64(200); key < 211; key++ { + keys = append(keys, key) + } + cdc := vectorindex.VectorIndexCdc[T]{ + Data: make([]vectorindex.VectorIndexCdcEntry[T], 0, len(keys)), + } v := []T{0.1, 0.2, 0.3} - // 0 - 199 key exists, 200 - 399 new insert - for i := 0; i < 400; i++ { + for _, key := range keys { e := vectorindex.VectorIndexCdcEntry[T]{Type: vectorindex.CDC_UPSERT, PKey: key, Vec: v} cdc.Data = append(cdc.Data, e) - key += 1 } - rand.Seed(uint64(time.Now().UnixNano())) - rand.Shuffle(len(cdc.Data), func(i, j int) { cdc.Data[i], cdc.Data[j] = cdc.Data[j], cdc.Data[i] }) + r := rand.New(rand.NewSource(99)) + r.Shuffle(len(cdc.Data), func(i, j int) { cdc.Data[i], cdc.Data[j] = cdc.Data[j], cdc.Data[i] }) var err error var sync *HnswSync[T] @@ -726,9 +742,17 @@ func runSyncContinuousUpdateInsertShuffle2FilesWithSmallCap[T types.RealNumbers] } defer sync.Destroy() - for i := 0; i < 10; i++ { + err = sync.Update(sqlproc, &cdc) + require.NoError(t, err) + require.Equal(t, int32(11), sync.ninsert.Load()) + require.Equal(t, int32(2), sync.nupdate.Load()) + require.Len(t, sync.indexes, 4) + + for cycle := 1; cycle < 10; cycle++ { err = sync.Update(sqlproc, &cdc) - require.Nil(t, err) + require.NoError(t, err, "update cycle %d", cycle+1) + require.Zero(t, sync.ninsert.Load(), "update cycle %d", cycle+1) + require.Equal(t, int32(len(cdc.Data)), sync.nupdate.Load(), "update cycle %d", cycle+1) } err = sync.Save(sqlproc) From 6f2e043994b0d5d7aba0f0e21e7a11cd57c9f855 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Fri, 28 Aug 2026 22:53:12 +0800 Subject: [PATCH 2/7] test: reduce race UT critical path --- .github/workflows/entrypoint.yaml | 9 +- Makefile | 2 + optools/run_ut.sh | 141 ++++++++++++++++++---------- pkg/vectorindex/hnsw/search_test.go | 10 +- pkg/vectorindex/hnsw/sync_test.go | 44 +++++++-- 5 files changed, 137 insertions(+), 69 deletions(-) diff --git a/.github/workflows/entrypoint.yaml b/.github/workflows/entrypoint.yaml index 444bb082b4a73..6ab4d2ad6ee96 100644 --- a/.github/workflows/entrypoint.yaml +++ b/.github/workflows/entrypoint.yaml @@ -73,9 +73,14 @@ jobs: name: Matrixone CI needs: check-pr-valid if: ${{ needs.check-pr-valid.outputs.pr_valid == 'true' && github.base_ref != '3.0-dev' }} - uses: matrixorigin/CI/.github/workflows/ci.yaml@main + # Use CI#438's opt-in shard contract for this canary. Switch back to @main + # after that prerequisite merges; the required-check summary is unchanged. + uses: matrixorigin/CI/.github/workflows/ci.yaml@codex/shard-ut-critical-path with: - ut_parallel: 8 + # Leave one runner CPU for race-detector/native work inside a package. + # Eight package slots made CPU-heavy HNSW tests the light-stage straggler. + ut_parallel: 7 + ut_sharded: true secrets: inherit matrixone-ut-coverage: diff --git a/Makefile b/Makefile index 967042d9fe2e6..ce3ee4af9d0d9 100644 --- a/Makefile +++ b/Makefile @@ -416,6 +416,8 @@ endif # bvt and unit test ############################################################################### UT_PARALLEL ?= 1 +UT_SHARD ?= all +export UT_SHARD # Native compilation runs before Go tests, so it can use an explicit UT CPU # budget without increasing peak race-test memory. With the default UT value, # omit -j and preserve recursive make's jobserver contract: a plain make stays diff --git a/optools/run_ut.sh b/optools/run_ut.sh index 4ec818c7bdeff..362b3bbb8fd6e 100755 --- a/optools/run_ut.sh +++ b/optools/run_ut.sh @@ -41,6 +41,7 @@ BUILD_WKSP=$(dirname "$PWD") && cd $BUILD_WKSP LOG="$G_TS-$TEST_TYPE.log" UT_TIMEOUT=${UT_TIMEOUT:-"15"} UT_PARALLEL=${UT_PARALLEL:-"1"} +UT_SHARD=${UT_SHARD:-"all"} HEAVY_RACE_PARALLEL=${HEAVY_RACE_PARALLEL:-"3"} PLAN_RACE_SHARDS=${PLAN_RACE_SHARDS:-"8"} # Two shards cut the measured engine/test race runtime roughly in half while @@ -481,6 +482,22 @@ function remove_packages_from_scope(){ printf '%s\n' "${scope}" } +function should_run_ut_stage(){ + local stage=$1 + + case "${UT_SHARD}:${stage}" in + all:* | \ + light-plan:light | light-plan:plan | \ + cluster:serial | cluster:embedded | \ + heavy:heavy) + return 0 + ;; + *) + return 1 + ;; + esac +} + function run_tests(){ cd $BUILD_WKSP horiz_rule @@ -490,10 +507,25 @@ function run_tests(){ echo "# COVERAGE REPORT: $CODE_COVERAGE" echo "# UT TIMEOUT: $UT_TIMEOUT" echo "# UT PARALLEL: $UT_PARALLEL" + echo "# UT SHARD: $UT_SHARD" echo "# CLUSTER ADMISSION: process lifecycle" echo "# HEAVY RACE UT: $HEAVY_RACE_PARALLEL total package slots" horiz_rule + case "${UT_SHARD}" in + all | light-plan | cluster | heavy) ;; + *) + logger "ERR" "UT_SHARD must be all, light-plan, cluster, or heavy; got '${UT_SHARD}'" + UT_TEST_STATUS=1 + return 0 + ;; + esac + if [[ "${SKIP_TESTS}" == "race" && "${UT_SHARD}" != "all" ]]; then + logger "ERR" "split UT shards require race mode; got SKIP_TESTS=race with UT_SHARD=${UT_SHARD}" + UT_TEST_STATUS=1 + return 0 + fi + logger "INF" "Clean go test cache" go clean -testcache @@ -564,13 +596,11 @@ function run_tests(){ return 0 fi - # These packages need exclusive runner access. NewTestService callers - # bind fixed ports, while the issues packages intentionally keep embedded - # clusters alive for most of their test processes. + # The issues packages intentionally keep embedded clusters alive for + # most of their test processes, so they retain exclusive runner access. + # The former logservice/TAE members of this group now allocate independent + # ports with collision retry and belong in the normal parallel scope. if ! serial_test_scope=$(go list ${GO_MODULE_MODE} \ - ./pkg/logservice \ - ./pkg/vm/engine/tae/logstore \ - ./pkg/vm/engine/tae/logstore/driver/logservicedriver \ ./pkg/tests/issues \ ./pkg/tests/issues/isolated); then logger "ERR" "Failed to resolve serial race-test packages" @@ -628,35 +658,38 @@ function run_tests(){ ${cluster_test_scope} \ ${resource_heavy_test_scope}) - if [[ -n "${light_test_scope}" ]]; then + : > "${UT_REPORT}" + if should_run_ut_stage light && [[ -n "${light_test_scope}" ]]; then logger "INF" "Run light race-test packages with parallelism ${UT_PARALLEL}" - LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p ${UT_PARALLEL} -timeout "${UT_TIMEOUT}m" -race $light_test_scope > $UT_REPORT + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p ${UT_PARALLEL} -timeout "${UT_TIMEOUT}m" -race $light_test_scope >> $UT_REPORT light_status=$? - else - : > "${UT_REPORT}" fi - logger "INF" "Run exclusive race-test packages serially" - for package in ${serial_test_scope}; do - logger "INF" "Run exclusive race-test package ${package}" - LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p 1 -timeout "${UT_TIMEOUT}m" -race "${package}" >> $UT_REPORT - package_status=$? - if (( package_status != 0 )); then - serial_status=1 - logger "ERR" "Exclusive race-test package ${package} failed with status ${package_status}" - fi - done + if should_run_ut_stage serial; then + logger "INF" "Run exclusive race-test packages serially" + for package in ${serial_test_scope}; do + logger "INF" "Run exclusive race-test package ${package}" + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p 1 -timeout "${UT_TIMEOUT}m" -race "${package}" >> $UT_REPORT + package_status=$? + if (( package_status != 0 )); then + serial_status=1 + logger "ERR" "Exclusive race-test package ${package} failed with status ${package_status}" + fi + done + fi # These packages link embedded clusters with substantial race-detector # memory. The runner-wide file-lock admission keeps complete cluster # lifecycles serialized across test binaries. Allow one additional # package process to overlap linking, setup, and non-cluster work without # returning to the six-way contention that starved HAKeeper. - logger "INF" "Run embedded-cluster race-test packages with package parallelism ${cluster_package_parallel} and serialized cluster lifecycle admission" - LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p "${cluster_package_parallel}" -timeout "${UT_TIMEOUT}m" -race $cluster_test_scope >> $UT_REPORT - cluster_status=$? + if should_run_ut_stage embedded; then + logger "INF" "Run embedded-cluster race-test packages with package parallelism ${cluster_package_parallel} and serialized cluster lifecycle admission" + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p "${cluster_package_parallel}" -timeout "${UT_TIMEOUT}m" -race $cluster_test_scope >> $UT_REPORT + cluster_status=$? + fi - if (( shard_engine == 1 )); then + if should_run_ut_stage heavy && (( shard_engine == 1 )); then # engine/test is dominated by serial fixture lifecycles inside one # process. Build it once and split every discovered top-level test # across fresh race processes. The effective shard count and the @@ -679,40 +712,44 @@ function run_tests(){ else resource_heavy_parallel=${HEAVY_RACE_PARALLEL} fi - else + elif should_run_ut_stage heavy; then resource_heavy_parallel=${HEAVY_RACE_PARALLEL} fi - logger "INF" "Run remaining resource-heavy race-test packages with parallelism ${resource_heavy_parallel}" - LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p ${resource_heavy_parallel} -timeout "${UT_TIMEOUT}m" -race $resource_heavy_test_scope >> $UT_REPORT - resource_heavy_status=$? - - if (( shard_engine == 1 )); then - if [[ -n "${ENGINE_RACE_JOB_PID}" ]]; then - wait "${ENGINE_RACE_JOB_PID}" - engine_status=$? - ENGINE_RACE_JOB_PID="" - else - # Keep the helper's process-group TERM trap scoped to a - # subshell even when a low budget requires sequential waves. - run_engine_race_shards "${engine_package}" "${engine_race_parallel}" & - ENGINE_RACE_JOB_PID=$! - wait "${ENGINE_RACE_JOB_PID}" - engine_status=$? - ENGINE_RACE_JOB_PID="" - fi - if [[ -s "${ENGINE_RACE_REPORT}" ]]; then - cat "${ENGINE_RACE_REPORT}" >> "${UT_REPORT}" + if should_run_ut_stage heavy; then + logger "INF" "Run remaining resource-heavy race-test packages with parallelism ${resource_heavy_parallel}" + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p ${resource_heavy_parallel} -timeout "${UT_TIMEOUT}m" -race $resource_heavy_test_scope >> $UT_REPORT + resource_heavy_status=$? + + if (( shard_engine == 1 )); then + if [[ -n "${ENGINE_RACE_JOB_PID}" ]]; then + wait "${ENGINE_RACE_JOB_PID}" + engine_status=$? + ENGINE_RACE_JOB_PID="" + else + # Keep the helper's process-group TERM trap scoped to a + # subshell even when a low budget requires sequential waves. + run_engine_race_shards "${engine_package}" "${engine_race_parallel}" & + ENGINE_RACE_JOB_PID=$! + wait "${ENGINE_RACE_JOB_PID}" + engine_status=$? + ENGINE_RACE_JOB_PID="" + fi + if [[ -s "${ENGINE_RACE_REPORT}" ]]; then + cat "${ENGINE_RACE_REPORT}" >> "${UT_REPORT}" + fi + rm -f "${ENGINE_RACE_TEST_BINARY}" "${ENGINE_RACE_REPORT}" "${ENGINE_RACE_REPORT}".* + ENGINE_RACE_TEST_BINARY="" + ENGINE_RACE_REPORT="" fi - rm -f "${ENGINE_RACE_TEST_BINARY}" "${ENGINE_RACE_REPORT}" "${ENGINE_RACE_REPORT}".* - ENGINE_RACE_TEST_BINARY="" - ENGINE_RACE_REPORT="" - fi - report_cgroup_memory_usage "Resource-heavy UT" + report_cgroup_memory_usage "Resource-heavy UT" + fi - run_plan_race_shards "${plan_package}" - plan_status=$? + if should_run_ut_stage plan; then + run_plan_race_shards "${plan_package}" + plan_status=$? + fi if (( light_status != 0 || serial_status != 0 || cluster_status != 0 || resource_heavy_status != 0 || engine_status != 0 || plan_status != 0 )); then UT_TEST_STATUS=1 diff --git a/pkg/vectorindex/hnsw/search_test.go b/pkg/vectorindex/hnsw/search_test.go index 516ec3ff7c42b..04b817526c4b7 100644 --- a/pkg/vectorindex/hnsw/search_test.go +++ b/pkg/vectorindex/hnsw/search_test.go @@ -314,11 +314,11 @@ func makeIndexBatch(proc *process.Process) *batch.Batch { } func TestFallocate(t *testing.T) { - - f, err := os.Create("apple") - require.Nil(t, err) - fallocate.Fallocate(f, 0, 10000) - f.Close() + f, err := os.CreateTemp(t.TempDir(), "fallocate-") + require.NoError(t, err) + t.Cleanup(func() { _ = f.Close() }) + require.NoError(t, fallocate.Fallocate(f, 0, 10000)) + require.NoError(t, f.Close()) } func makeMetaBatch2Files(proc *process.Process) *batch.Batch { diff --git a/pkg/vectorindex/hnsw/sync_test.go b/pkg/vectorindex/hnsw/sync_test.go index 512dc079b4cc1..67f2d0654e8dd 100644 --- a/pkg/vectorindex/hnsw/sync_test.go +++ b/pkg/vectorindex/hnsw/sync_test.go @@ -691,25 +691,41 @@ func runSyncContinuousUpdateInsertShuffle2FilesWithSmallCap[T types.RealNumbers] } }) + oldRunSQL := runSql + oldRunSQLStreaming := runSql_streaming + oldRunTxn := runTxn + t.Cleanup(func() { + runSql = oldRunSQL + runSql_streaming = oldRunSQLStreaming + runTxn = oldRunTxn + }) runSql = mock_runSql_2files runSql_streaming = mock_runSql_streaming_2files runTxn = mock_runTxn indexes := mockMoIndexes() - cdc := vectorindex.VectorIndexCdc[T]{Data: make([]vectorindex.VectorIndexCdcEntry[T], 0, 100)} - - key := int64(0) + // The fixture has two existing files containing keys 0..199. Exercise one + // update in each file plus eleven inserts: at capacity ten, the inserts must + // roll over into two new models. Thirteen entries also keep all eight build + // workers active. Preserve the original ten Update cycles as repeated + // lifecycle/stability coverage, while removing entries that only duplicated + // work inside each cycle and became prohibitively expensive under -race. + keys := []int64{0, 100} + for key := int64(200); key < 211; key++ { + keys = append(keys, key) + } + cdc := vectorindex.VectorIndexCdc[T]{ + Data: make([]vectorindex.VectorIndexCdcEntry[T], 0, len(keys)), + } v := []T{0.1, 0.2, 0.3} - // 0 - 199 key exists, 200 - 399 new insert - for i := 0; i < 400; i++ { + for _, key := range keys { e := vectorindex.VectorIndexCdcEntry[T]{Type: vectorindex.CDC_UPSERT, PKey: key, Vec: v} cdc.Data = append(cdc.Data, e) - key += 1 } - rand.Seed(uint64(time.Now().UnixNano())) - rand.Shuffle(len(cdc.Data), func(i, j int) { cdc.Data[i], cdc.Data[j] = cdc.Data[j], cdc.Data[i] }) + r := rand.New(rand.NewSource(99)) + r.Shuffle(len(cdc.Data), func(i, j int) { cdc.Data[i], cdc.Data[j] = cdc.Data[j], cdc.Data[i] }) var err error var sync *HnswSync[T] @@ -726,9 +742,17 @@ func runSyncContinuousUpdateInsertShuffle2FilesWithSmallCap[T types.RealNumbers] } defer sync.Destroy() - for i := 0; i < 10; i++ { + err = sync.Update(sqlproc, &cdc) + require.NoError(t, err) + require.Equal(t, int32(11), sync.ninsert.Load()) + require.Equal(t, int32(2), sync.nupdate.Load()) + require.Len(t, sync.indexes, 4) + + for cycle := 1; cycle < 10; cycle++ { err = sync.Update(sqlproc, &cdc) - require.Nil(t, err) + require.NoError(t, err, "update cycle %d", cycle+1) + require.Zero(t, sync.ninsert.Load(), "update cycle %d", cycle+1) + require.Equal(t, int32(len(cdc.Data)), sync.nupdate.Load(), "update cycle %d", cycle+1) } err = sync.Save(sqlproc) From ed055de40a30f1b77538f435400ff5e7b5bc8a3d Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Fri, 28 Aug 2026 23:11:12 +0800 Subject: [PATCH 3/7] ci: balance race UT shards --- optools/run_ut.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/optools/run_ut.sh b/optools/run_ut.sh index 362b3bbb8fd6e..973d18fcf5ba1 100755 --- a/optools/run_ut.sh +++ b/optools/run_ut.sh @@ -487,9 +487,10 @@ function should_run_ut_stage(){ case "${UT_SHARD}:${stage}" in all:* | \ - light-plan:light | light-plan:plan | \ - cluster:serial | cluster:embedded | \ - heavy:heavy) + light:light | \ + issues:serial | \ + embedded:embedded | \ + heavy-plan:heavy | heavy-plan:plan) return 0 ;; *) @@ -513,9 +514,9 @@ function run_tests(){ horiz_rule case "${UT_SHARD}" in - all | light-plan | cluster | heavy) ;; + all | light | issues | embedded | heavy-plan) ;; *) - logger "ERR" "UT_SHARD must be all, light-plan, cluster, or heavy; got '${UT_SHARD}'" + logger "ERR" "UT_SHARD must be all, light, issues, embedded, or heavy-plan; got '${UT_SHARD}'" UT_TEST_STATUS=1 return 0 ;; From 29d0ac136116a122b6ef99e154b70f1289cee465 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Fri, 28 Aug 2026 23:21:01 +0800 Subject: [PATCH 4/7] test: preserve HNSW intent with orthogonal coverage --- optools/run_ut.sh | 24 ++++++++++++++++++++++-- pkg/vectorindex/hnsw/sync_test.go | 20 ++++++++++++-------- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/optools/run_ut.sh b/optools/run_ut.sh index 973d18fcf5ba1..b25de5c4e2ea5 100755 --- a/optools/run_ut.sh +++ b/optools/run_ut.sh @@ -487,7 +487,7 @@ function should_run_ut_stage(){ case "${UT_SHARD}:${stage}" in all:* | \ - light:light | \ + light:light | light:hnsw | \ issues:serial | \ embedded:embedded | \ heavy-plan:heavy | heavy-plan:plan) @@ -559,6 +559,7 @@ function run_tests(){ logger "INF" "Run UT with race check" local plan_package local engine_package + local hnsw_package local serial_test_scope local cluster_test_scope local resource_heavy_test_scope @@ -567,6 +568,7 @@ function run_tests(){ local cluster_package_parallel=2 local package_status=0 local light_status=0 + local hnsw_status=0 local serial_status=0 local cluster_status=0 local resource_heavy_status=0 @@ -596,6 +598,11 @@ function run_tests(){ UT_TEST_STATUS=1 return 0 fi + if ! hnsw_package=$(go list ${GO_MODULE_MODE} ./pkg/vectorindex/hnsw); then + logger "ERR" "Failed to resolve ./pkg/vectorindex/hnsw" + UT_TEST_STATUS=1 + return 0 + fi # The issues packages intentionally keep embedded clusters alive for # most of their test processes, so they retain exclusive runner access. @@ -624,6 +631,7 @@ function run_tests(){ cluster_test_scope=$(remove_packages_from_scope \ "${cluster_test_scope}" \ "${plan_package}" \ + "${hnsw_package}" \ ${serial_test_scope}) # Dependency-based group precedence remains authoritative. If this @@ -648,6 +656,7 @@ function run_tests(){ "${resource_heavy_test_scope}" \ "${plan_package}" \ "${engine_package}" \ + "${hnsw_package}" \ ${serial_test_scope} \ ${cluster_test_scope}) @@ -655,6 +664,7 @@ function run_tests(){ "${test_scope}" \ "${plan_package}" \ "${engine_package}" \ + "${hnsw_package}" \ ${serial_test_scope} \ ${cluster_test_scope} \ ${resource_heavy_test_scope}) @@ -666,6 +676,16 @@ function run_tests(){ light_status=$? fi + # HNSW owns native worker pools inside its test binary. Running it as + # one package slot after the normal light wave preserves its full-batch + # and repeated-lifecycle targets without letting package concurrency + # turn CPU scheduling delay into a multi-minute light-stage straggler. + if should_run_ut_stage hnsw; then + logger "INF" "Run HNSW race-test package with exclusive runner CPU" + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test ${GO_MODULE_MODE} ${GO_TEST_VET_FLAGS} -short -v -json -tags "${TAGS}" -p 1 -timeout "${UT_TIMEOUT}m" -race "${hnsw_package}" >> $UT_REPORT + hnsw_status=$? + fi + if should_run_ut_stage serial; then logger "INF" "Run exclusive race-test packages serially" for package in ${serial_test_scope}; do @@ -752,7 +772,7 @@ function run_tests(){ plan_status=$? fi - if (( light_status != 0 || serial_status != 0 || cluster_status != 0 || resource_heavy_status != 0 || engine_status != 0 || plan_status != 0 )); then + if (( light_status != 0 || hnsw_status != 0 || serial_status != 0 || cluster_status != 0 || resource_heavy_status != 0 || engine_status != 0 || plan_status != 0 )); then UT_TEST_STATUS=1 fi fi diff --git a/pkg/vectorindex/hnsw/sync_test.go b/pkg/vectorindex/hnsw/sync_test.go index 67f2d0654e8dd..c69319e985bde 100644 --- a/pkg/vectorindex/hnsw/sync_test.go +++ b/pkg/vectorindex/hnsw/sync_test.go @@ -528,8 +528,10 @@ func TestSyncDeleteShuffle2Files(t *testing.T) { key += 1 } - rand.Seed(uint64(time.Now().UnixNano())) - rand.Shuffle(len(cdc.Data), func(i, j int) { cdc.Data[i], cdc.Data[j] = cdc.Data[j], cdc.Data[i] }) + seed := uint64(time.Now().UnixNano()) + t.Logf("CDC shuffle seed: %d", seed) + r := rand.New(rand.NewSource(seed)) + r.Shuffle(len(cdc.Data), func(i, j int) { cdc.Data[i], cdc.Data[j] = cdc.Data[j], cdc.Data[i] }) sync, err := NewHnswSync[float32](sqlproc, "db", "src", "idx", indexes, int32(types.T_array_float32), 3) require.Nil(t, err) @@ -704,12 +706,10 @@ func runSyncContinuousUpdateInsertShuffle2FilesWithSmallCap[T types.RealNumbers] runTxn = mock_runTxn indexes := mockMoIndexes() - // The fixture has two existing files containing keys 0..199. Exercise one - // update in each file plus eleven inserts: at capacity ten, the inserts must - // roll over into two new models. Thirteen entries also keep all eight build - // workers active. Preserve the original ten Update cycles as repeated - // lifecycle/stability coverage, while removing entries that only duplicated - // work inside each cycle and became prohibitively expensive under -race. + // The preceding one-shot SmallCap test retains the full 400-row mixed batch. + // This test owns the orthogonal repeated-lifecycle target: one update from + // each existing file plus eleven inserts cross the capacity-10 boundary into + // two models. Thirteen rows also distribute work to all eight build workers. keys := []int64{0, 100} for key := int64(200); key < 211; key++ { keys = append(keys, key) @@ -724,6 +724,8 @@ func runSyncContinuousUpdateInsertShuffle2FilesWithSmallCap[T types.RealNumbers] cdc.Data = append(cdc.Data, e) } + // Keep this lifecycle regression reproducible; the adjacent full-batch test + // retains time-seeded shuffle diversity. r := rand.New(rand.NewSource(99)) r.Shuffle(len(cdc.Data), func(i, j int) { cdc.Data[i], cdc.Data[j] = cdc.Data[j], cdc.Data[i] }) @@ -747,6 +749,8 @@ func runSyncContinuousUpdateInsertShuffle2FilesWithSmallCap[T types.RealNumbers] require.Equal(t, int32(11), sync.ninsert.Load()) require.Equal(t, int32(2), sync.nupdate.Load()) require.Len(t, sync.indexes, 4) + require.Equal(t, int64(10), sync.indexes[2].Len.Load()) + require.Equal(t, int64(1), sync.indexes[3].Len.Load()) for cycle := 1; cycle < 10; cycle++ { err = sync.Update(sqlproc, &cdc) From 8df15f350a52dd291f2e14fbd79469a81dce9145 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Fri, 28 Aug 2026 23:49:15 +0800 Subject: [PATCH 5/7] test: make sharded UT routing fail closed --- .github/workflows/entrypoint.yaml | 6 +- Makefile | 7 +- optools/run_ut.sh | 86 +++++++++++++++--------- optools/ut_tools.bash | 102 ++++++++++++++++++++++++++++ optools/ut_tools_test.go | 107 ++++++++++++++++++++++++++++++ 5 files changed, 270 insertions(+), 38 deletions(-) diff --git a/.github/workflows/entrypoint.yaml b/.github/workflows/entrypoint.yaml index 6ab4d2ad6ee96..1746a590dbb4a 100644 --- a/.github/workflows/entrypoint.yaml +++ b/.github/workflows/entrypoint.yaml @@ -73,9 +73,9 @@ jobs: name: Matrixone CI needs: check-pr-valid if: ${{ needs.check-pr-valid.outputs.pr_valid == 'true' && github.base_ref != '3.0-dev' }} - # Use CI#438's opt-in shard contract for this canary. Switch back to @main - # after that prerequisite merges; the required-check summary is unchanged. - uses: matrixorigin/CI/.github/workflows/ci.yaml@codex/shard-ut-critical-path + # Pin CI#438's reviewed shard contract for this canary. Switch back to + # @main after that prerequisite merges; the required-check name is stable. + uses: matrixorigin/CI/.github/workflows/ci.yaml@a1c554b95955300d6407892833a6625c0021d52d with: # Leave one runner CPU for race-detector/native work inside a package. # Eight package slots made CPU-heavy HNSW tests the light-stage straggler. diff --git a/Makefile b/Makefile index ce3ee4af9d0d9..e7f3c6242e934 100644 --- a/Makefile +++ b/Makefile @@ -406,9 +406,10 @@ ut: $(UT_PREREQUISITES) ifeq ($(UNAME_S),darwin) @cd optools && ./run_ut.sh UT $(SKIP_TEST) else - # The race suite is split into light, exclusive, heavy, and plan shards. - # Keep the outer budget above the per-package timeout so an expanded main - # branch cannot be killed while later shards are still making progress. + # The race suite is internally partitioned into light/HNSW, exclusive issues, + # embedded-cluster, heavy/engine, and plan stages. Keep the outer budget above + # the per-package timeout so an expanded main branch cannot be killed while a + # selected stage is still making progress. @cd optools && timeout 90m ./run_ut.sh UT $(SKIP_TEST) endif diff --git a/optools/run_ut.sh b/optools/run_ut.sh index b25de5c4e2ea5..9131578455c07 100755 --- a/optools/run_ut.sh +++ b/optools/run_ut.sh @@ -482,23 +482,6 @@ function remove_packages_from_scope(){ printf '%s\n' "${scope}" } -function should_run_ut_stage(){ - local stage=$1 - - case "${UT_SHARD}:${stage}" in - all:* | \ - light:light | light:hnsw | \ - issues:serial | \ - embedded:embedded | \ - heavy-plan:heavy | heavy-plan:plan) - return 0 - ;; - *) - return 1 - ;; - esac -} - function run_tests(){ cd $BUILD_WKSP horiz_rule @@ -513,14 +496,22 @@ function run_tests(){ echo "# HEAVY RACE UT: $HEAVY_RACE_PARALLEL total package slots" horiz_rule - case "${UT_SHARD}" in - all | light | issues | embedded | heavy-plan) ;; - *) - logger "ERR" "UT_SHARD must be all, light, issues, embedded, or heavy-plan; got '${UT_SHARD}'" - UT_TEST_STATUS=1 - return 0 - ;; - esac + if ! list_ut_shard_stages "${UT_SHARD}" >/dev/null; then + logger "ERR" "UT_SHARD must be all, light, issues, embedded, or heavy-plan; got '${UT_SHARD}'" + UT_TEST_STATUS=1 + return 0 + fi + if ! validate_complete_partition \ + "UT stage" \ + "$(list_ut_shard_stages all)" \ + "$(list_ut_shard_stages light)" \ + "$(list_ut_shard_stages issues)" \ + "$(list_ut_shard_stages embedded)" \ + "$(list_ut_shard_stages heavy-plan)"; then + logger "ERR" "Race-UT shard stages are not a complete disjoint partition" + UT_TEST_STATUS=1 + return 0 + fi if [[ "${SKIP_TESTS}" == "race" && "${UT_SHARD}" != "all" ]]; then logger "ERR" "split UT shards require race mode; got SKIP_TESTS=race with UT_SHARD=${UT_SHARD}" UT_TEST_STATUS=1 @@ -530,7 +521,18 @@ function run_tests(){ logger "INF" "Clean go test cache" go clean -testcache - local test_scope=$(go list ${GO_MODULE_MODE} ./... | grep -v 'driver/aoe' | grep -v 'engine/aoe' | grep -v 'pkg/catalog') + local test_scope + if ! test_scope=$(go list ${GO_MODULE_MODE} ./...); then + logger "ERR" "Failed to resolve the complete UT package scope" + UT_TEST_STATUS=1 + return 0 + fi + test_scope=$(printf '%s\n' "${test_scope}" | grep -v 'driver/aoe' | grep -v 'engine/aoe' | grep -v 'pkg/catalog') + if [[ -z "${test_scope}" ]]; then + logger "ERR" "The complete UT package scope is empty" + UT_TEST_STATUS=1 + return 0 + fi local leave_out=$(egrep -lr --include="*.go" 'Code generated by protoc-gen-gogo. DO NOT EDIT.' ./pkg/* | sort -u | xargs basename -a) logger "INF" "Ingore code coverage $(echo ${leave_out[@]}|tr " " "|")" local cover_profile='profile.raw' @@ -560,6 +562,7 @@ function run_tests(){ local plan_package local engine_package local hnsw_package + local engine_test_scope local serial_test_scope local cluster_test_scope local resource_heavy_test_scope @@ -604,13 +607,14 @@ function run_tests(){ return 0 fi - # The issues packages intentionally keep embedded clusters alive for - # most of their test processes, so they retain exclusive runner access. - # The former logservice/TAE members of this group now allocate independent - # ports with collision retry and belong in the normal parallel scope. + # The main issues package keeps its shared base cluster alive for most + # of the test process, so it retains an exclusive runner. The isolated + # issues package has no shared base and safely belongs to the embedded + # group: runner-wide admission still serializes each complete cluster + # lifecycle. Former logservice/TAE members allocate independent ports + # with collision retry and belong in the normal parallel scope. if ! serial_test_scope=$(go list ${GO_MODULE_MODE} \ - ./pkg/tests/issues \ - ./pkg/tests/issues/isolated); then + ./pkg/tests/issues); then logger "ERR" "Failed to resolve serial race-test packages" UT_TEST_STATUS=1 return 0 @@ -640,7 +644,10 @@ function run_tests(){ if printf '%s\n%s\n' "${serial_test_scope}" "${cluster_test_scope}" | grep -Fxq "${engine_package}"; then shard_engine=0 + engine_test_scope="" logger "INF" "Keep ${engine_package} in its higher-precedence race-test group" + else + engine_test_scope="${engine_package}" fi if ! resource_heavy_test_scope=$(go list ${GO_MODULE_MODE} \ @@ -669,6 +676,21 @@ function run_tests(){ ${cluster_test_scope} \ ${resource_heavy_test_scope}) + if ! validate_complete_partition \ + "UT package" \ + "${test_scope}" \ + "${light_test_scope}" \ + "${hnsw_package}" \ + "${serial_test_scope}" \ + "${cluster_test_scope}" \ + "${resource_heavy_test_scope}" \ + "${engine_test_scope}" \ + "${plan_package}"; then + logger "ERR" "Race-UT package groups are not a complete disjoint partition" + UT_TEST_STATUS=1 + return 0 + fi + : > "${UT_REPORT}" if should_run_ut_stage light && [[ -n "${light_test_scope}" ]]; then logger "INF" "Run light race-test packages with parallelism ${UT_PARALLEL}" diff --git a/optools/ut_tools.bash b/optools/ut_tools.bash index 7169ae1b2cddf..d8f9ffb41d1ac 100644 --- a/optools/ut_tools.bash +++ b/optools/ut_tools.bash @@ -91,3 +91,105 @@ function list_embedded_cluster_test_packages() { printf '%s\n' "${discovered_packages}" | sed '/^$/d' | LC_ALL=C sort -u } + +# list_ut_shard_stages is the single source of truth for the race-UT shard +# contract. Keep the all-suite path explicit so a newly introduced stage cannot +# run only in UT_SHARD=all while being silently absent from every CI shard. +function list_ut_shard_stages() { + if (( $# != 1 )); then + echo "Usage: list_ut_shard_stages SHARD" >&2 + return 2 + fi + + case "$1" in + all) + printf '%s\n' light hnsw serial embedded heavy plan + ;; + light) + printf '%s\n' light hnsw + ;; + issues) + printf '%s\n' serial + ;; + embedded) + printf '%s\n' embedded + ;; + heavy-plan) + printf '%s\n' heavy plan + ;; + *) + echo "Unknown UT shard '$1'" >&2 + return 2 + ;; + esac +} + +function should_run_ut_stage() { + if (( $# != 1 )); then + echo "Usage: should_run_ut_stage STAGE" >&2 + return 2 + fi + + if ! list_ut_shard_stages all | grep -Fxq "$1"; then + echo "Unknown UT stage '$1'" >&2 + return 2 + fi + + list_ut_shard_stages "${UT_SHARD:-all}" | grep -Fxq "$1" +} + +# validate_complete_partition proves that the supplied groups are a disjoint, +# complete partition of the authoritative item scope. It is cheap enough to run +# before every shard and makes routing or discovery drift fail closed instead of +# producing a green run with missing or duplicated coverage. +function validate_complete_partition() { + if (( $# < 3 )); then + echo "Usage: validate_complete_partition LABEL EXPECTED GROUP [GROUP...]" >&2 + return 2 + fi + + local label=$1 + local expected=$2 + shift 2 + local group + local package + + { + while IFS= read -r package; do + if [[ -n "${package}" ]]; then + printf 'expected\t%s\n' "${package}" + fi + done <<< "${expected}" + + for group in "$@"; do + while IFS= read -r package; do + if [[ -n "${package}" ]]; then + printf 'actual\t%s\n' "${package}" + fi + done <<< "${group}" + done + } | awk -F '\t' -v label="${label}" ' + $1 == "expected" { expected[$2] = 1; next } + $1 == "actual" { actual[$2]++; next } + END { + failed = 0 + for (package in expected) { + if (!(package in actual)) { + print "Missing " label " from partition: " package > "/dev/stderr" + failed = 1 + } + } + for (package in actual) { + if (!(package in expected)) { + print "Unexpected " label " in partition: " package > "/dev/stderr" + failed = 1 + } + if (actual[package] != 1) { + print label " occurs " actual[package] " times in partition: " package > "/dev/stderr" + failed = 1 + } + } + exit failed + } + ' +} diff --git a/optools/ut_tools_test.go b/optools/ut_tools_test.go index 8cff351f28651..187261b059747 100644 --- a/optools/ut_tools_test.go +++ b/optools/ut_tools_test.go @@ -259,3 +259,110 @@ func TestListEmbeddedClusterTestPackagesPreservesGoListFailure(t *testing.T) { t.Fatalf("expected one go list attempt, got %q", attempts) } } + +func runUTToolsBash(t *testing.T, script string, env ...string) ([]byte, int) { + t.Helper() + + toolsPath, err := filepath.Abs("ut_tools.bash") + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("bash", "-c", script, "bash", toolsPath) + cmd.Env = append(os.Environ(), env...) + output, err := cmd.CombinedOutput() + if err == nil { + return output, 0 + } + if exitError, ok := err.(*exec.ExitError); ok { + return output, exitError.ExitCode() + } + t.Fatalf("run UT tools bash: %v", err) + return nil, 0 +} + +func TestUTShardStagesFormCompleteDisjointMap(t *testing.T) { + script := `source "$1" +for shard in all light issues embedded heavy-plan; do + stages=$(list_ut_shard_stages "${shard}") || exit $? + printf '%s=' "${shard}" + printf '%s\n' "${stages}" | paste -sd, - +done +for shard in light issues embedded heavy-plan; do + UT_SHARD=${shard} + for stage in light hnsw serial embedded heavy plan; do + if should_run_ut_stage "${stage}"; then + printf '%s:%s\n' "${shard}" "${stage}" + else + status=$? + if (( status != 1 )); then exit "${status}"; fi + fi + done +done` + output, status := runUTToolsBash(t, script) + if status != 0 { + t.Fatalf("shard map failed with status %d: %s", status, output) + } + + expected := `all=light,hnsw,serial,embedded,heavy,plan +light=light,hnsw +issues=serial +embedded=embedded +heavy-plan=heavy,plan +light:light +light:hnsw +issues:serial +embedded:embedded +heavy-plan:heavy +heavy-plan:plan` + if actual := strings.TrimSpace(string(output)); actual != expected { + t.Fatalf("unexpected shard map:\n%s\nexpected:\n%s", actual, expected) + } +} + +func TestUTShardStagesRejectUnknownValues(t *testing.T) { + tests := []struct { + name string + script string + }{ + {name: "shard", script: `source "$1"; list_ut_shard_stages unknown`}, + {name: "stage", script: `source "$1"; UT_SHARD=all; should_run_ut_stage unknown`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + output, status := runUTToolsBash(t, test.script) + if status != 2 { + t.Fatalf("expected status 2, got %d: %s", status, output) + } + }) + } +} + +func TestValidateCompletePartition(t *testing.T) { + tests := []struct { + name string + groups []string + wantStatus int + wantError string + }{ + {name: "complete", groups: []string{"a", "b\nc"}}, + {name: "missing", groups: []string{"a", "b"}, wantStatus: 1, wantError: "Missing UT package from partition: c"}, + {name: "duplicate", groups: []string{"a\nb", "b\nc"}, wantStatus: 1, wantError: "UT package occurs 2 times in partition: b"}, + {name: "unexpected", groups: []string{"a\nb", "c\nd"}, wantStatus: 1, wantError: "Unexpected UT package in partition: d"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + script := `source "$1"; validate_complete_partition "UT package" "${EXPECTED}" "${GROUP_ONE}" "${GROUP_TWO}"` + output, status := runUTToolsBash(t, script, + "EXPECTED=a\nb\nc", + "GROUP_ONE="+test.groups[0], + "GROUP_TWO="+test.groups[1], + ) + if status != test.wantStatus { + t.Fatalf("expected status %d, got %d: %s", test.wantStatus, status, output) + } + if test.wantError != "" && !strings.Contains(string(output), test.wantError) { + t.Fatalf("missing error %q in output: %s", test.wantError, output) + } + }) + } +} From f5ef3fcda3cf63854e5c18cbee17239ef21a7448 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sat, 29 Aug 2026 03:33:44 +0800 Subject: [PATCH 6/7] test: harden sharded UT revision and routing --- .github/workflows/entrypoint.yaml | 2 +- optools/run_ut.sh | 3 ++- optools/ut_tools.bash | 15 +++++++++++++-- optools/ut_tools_test.go | 11 +++++++++-- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/entrypoint.yaml b/.github/workflows/entrypoint.yaml index 1746a590dbb4a..bc7c39b87f691 100644 --- a/.github/workflows/entrypoint.yaml +++ b/.github/workflows/entrypoint.yaml @@ -75,7 +75,7 @@ jobs: if: ${{ needs.check-pr-valid.outputs.pr_valid == 'true' && github.base_ref != '3.0-dev' }} # Pin CI#438's reviewed shard contract for this canary. Switch back to # @main after that prerequisite merges; the required-check name is stable. - uses: matrixorigin/CI/.github/workflows/ci.yaml@a1c554b95955300d6407892833a6625c0021d52d + uses: matrixorigin/CI/.github/workflows/ci.yaml@d303928bb44b24e56ee594f812b582e43be8c854 with: # Leave one runner CPU for race-detector/native work inside a package. # Eight package slots made CPU-heavy HNSW tests the light-stage straggler. diff --git a/optools/run_ut.sh b/optools/run_ut.sh index 9131578455c07..cdea460821807 100755 --- a/optools/run_ut.sh +++ b/optools/run_ut.sh @@ -55,6 +55,7 @@ CODE_COVERAGE="$G_WKSP/$G_TS-UT-Coverage.html" RAW_COVERAGE="coverage.out" IS_BUILD_FAIL="" UT_TEST_STATUS=0 +UT_SHARD_ROUTING_ERROR=0 PLAN_RACE_TEST_BINARY="" ENGINE_RACE_TEST_BINARY="" ENGINE_RACE_JOB_PID="" @@ -794,7 +795,7 @@ function run_tests(){ plan_status=$? fi - if (( light_status != 0 || hnsw_status != 0 || serial_status != 0 || cluster_status != 0 || resource_heavy_status != 0 || engine_status != 0 || plan_status != 0 )); then + if (( UT_SHARD_ROUTING_ERROR != 0 || light_status != 0 || hnsw_status != 0 || serial_status != 0 || cluster_status != 0 || resource_heavy_status != 0 || engine_status != 0 || plan_status != 0 )); then UT_TEST_STATUS=1 fi fi diff --git a/optools/ut_tools.bash b/optools/ut_tools.bash index d8f9ffb41d1ac..3166f1ef371ba 100644 --- a/optools/ut_tools.bash +++ b/optools/ut_tools.bash @@ -127,15 +127,22 @@ function list_ut_shard_stages() { function should_run_ut_stage() { if (( $# != 1 )); then echo "Usage: should_run_ut_stage STAGE" >&2 + UT_SHARD_ROUTING_ERROR=1 return 2 fi if ! list_ut_shard_stages all | grep -Fxq "$1"; then echo "Unknown UT stage '$1'" >&2 + UT_SHARD_ROUTING_ERROR=1 return 2 fi - list_ut_shard_stages "${UT_SHARD:-all}" | grep -Fxq "$1" + local selected_stages + if ! selected_stages=$(list_ut_shard_stages "${UT_SHARD:-all}"); then + UT_SHARD_ROUTING_ERROR=1 + return 2 + fi + printf '%s\n' "${selected_stages}" | grep -Fxq "$1" } # validate_complete_partition proves that the supplied groups are a disjoint, @@ -169,11 +176,15 @@ function validate_complete_partition() { done <<< "${group}" done } | awk -F '\t' -v label="${label}" ' - $1 == "expected" { expected[$2] = 1; next } + $1 == "expected" { expected[$2]++; next } $1 == "actual" { actual[$2]++; next } END { failed = 0 for (package in expected) { + if (expected[package] != 1) { + print label " occurs " expected[package] " times in expected scope: " package > "/dev/stderr" + failed = 1 + } if (!(package in actual)) { print "Missing " label " from partition: " package > "/dev/stderr" failed = 1 diff --git a/optools/ut_tools_test.go b/optools/ut_tools_test.go index 187261b059747..789c3b15e5e7c 100644 --- a/optools/ut_tools_test.go +++ b/optools/ut_tools_test.go @@ -325,7 +325,8 @@ func TestUTShardStagesRejectUnknownValues(t *testing.T) { script string }{ {name: "shard", script: `source "$1"; list_ut_shard_stages unknown`}, - {name: "stage", script: `source "$1"; UT_SHARD=all; should_run_ut_stage unknown`}, + {name: "stage", script: `source "$1"; UT_SHARD=all; UT_SHARD_ROUTING_ERROR=0; should_run_ut_stage unknown; status=$?; (( UT_SHARD_ROUTING_ERROR == 1 )) || exit 3; exit "${status}"`}, + {name: "selected shard", script: `source "$1"; UT_SHARD=unknown; UT_SHARD_ROUTING_ERROR=0; should_run_ut_stage light; status=$?; (( UT_SHARD_ROUTING_ERROR == 1 )) || exit 3; exit "${status}"`}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -340,20 +341,26 @@ func TestUTShardStagesRejectUnknownValues(t *testing.T) { func TestValidateCompletePartition(t *testing.T) { tests := []struct { name string + expected string groups []string wantStatus int wantError string }{ {name: "complete", groups: []string{"a", "b\nc"}}, + {name: "duplicate expected", expected: "a\nb\nb\nc", groups: []string{"a", "b\nc"}, wantStatus: 1, wantError: "UT package occurs 2 times in expected scope: b"}, {name: "missing", groups: []string{"a", "b"}, wantStatus: 1, wantError: "Missing UT package from partition: c"}, {name: "duplicate", groups: []string{"a\nb", "b\nc"}, wantStatus: 1, wantError: "UT package occurs 2 times in partition: b"}, {name: "unexpected", groups: []string{"a\nb", "c\nd"}, wantStatus: 1, wantError: "Unexpected UT package in partition: d"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { + expected := test.expected + if expected == "" { + expected = "a\nb\nc" + } script := `source "$1"; validate_complete_partition "UT package" "${EXPECTED}" "${GROUP_ONE}" "${GROUP_TWO}"` output, status := runUTToolsBash(t, script, - "EXPECTED=a\nb\nc", + "EXPECTED="+expected, "GROUP_ONE="+test.groups[0], "GROUP_TWO="+test.groups[1], ) From 8602561e897467b2e4def85e9607a9dc87a7890e Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sat, 29 Aug 2026 03:52:41 +0800 Subject: [PATCH 7/7] ci: use merged UT sharding workflow --- .github/workflows/entrypoint.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/entrypoint.yaml b/.github/workflows/entrypoint.yaml index bc7c39b87f691..025e6b78641b9 100644 --- a/.github/workflows/entrypoint.yaml +++ b/.github/workflows/entrypoint.yaml @@ -73,9 +73,7 @@ jobs: name: Matrixone CI needs: check-pr-valid if: ${{ needs.check-pr-valid.outputs.pr_valid == 'true' && github.base_ref != '3.0-dev' }} - # Pin CI#438's reviewed shard contract for this canary. Switch back to - # @main after that prerequisite merges; the required-check name is stable. - uses: matrixorigin/CI/.github/workflows/ci.yaml@d303928bb44b24e56ee594f812b582e43be8c854 + uses: matrixorigin/CI/.github/workflows/ci.yaml@main with: # Leave one runner CPU for race-detector/native work inside a package. # Eight package slots made CPU-heavy HNSW tests the light-stage straggler.